Compare commits
48
Commits
14ad2fd0f8
..
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
98121e6162 | ||
|
|
e447a0b6da | ||
|
|
530d45448a | ||
|
|
6a0759693c | ||
|
|
83c78ade5d | ||
|
|
eebe8eef89 | ||
|
|
a22a7288c1 | ||
|
|
40d56c515d | ||
|
|
b8ad6bb10b | ||
|
|
131f744745 | ||
|
|
fef686f977 | ||
|
|
4ede0724a7 | ||
|
|
03a5ce5bb1 | ||
|
|
745aa023a8 | ||
|
|
5633aa2efa | ||
|
|
6643185a81 | ||
|
|
d0149e70a7 | ||
|
|
84ac366e36 | ||
|
|
b700e05c5e | ||
|
|
9568691609 | ||
|
|
35404bc7a4 | ||
|
|
6a95d16a4d | ||
|
|
39f1a5c578 | ||
|
|
09eaf60a90 | ||
|
|
4ad8ab4f1b | ||
|
|
27743535c5 | ||
|
|
3f9695a0ed | ||
|
|
b1836b61c8 | ||
|
|
a62cfcac3d | ||
|
|
5b9c635ed1 | ||
|
|
ad56e7b9e0 | ||
|
|
4eb6840c2a | ||
|
|
9f173d7aee | ||
|
|
5f0320fde0 | ||
|
|
d21ba65a43 | ||
|
|
ea61b45a87 | ||
|
|
f34de8d1b0 | ||
|
|
f1898e6942 | ||
|
|
8970757a8e | ||
|
|
c55953769f | ||
|
|
cf8f1a0f7b | ||
|
|
1efa4715d7 | ||
|
|
ec307112a9 | ||
|
|
8f0609ade2 | ||
|
|
327f255c9d | ||
|
|
06a5f5bec7 | ||
|
|
00ba8b9dd3 | ||
|
|
d7e0171b2f |
+1
-1
Submodule common updated: ec0812ad9d...7df69021ea
@@ -40,6 +40,7 @@ exports.up = async (knex) => {
|
||||
.index();
|
||||
|
||||
table.text('name');
|
||||
table.string('name_stylized');
|
||||
table.text('slug', 32);
|
||||
|
||||
table.text('type')
|
||||
@@ -126,10 +127,15 @@ exports.up = async (knex) => {
|
||||
.defaultTo(knex.fn.now());
|
||||
});
|
||||
|
||||
await knex.schema.createMaterializedView('media_sfw', (view) => {
|
||||
view.as(knex('media').select('id').where('is_sfw', true));
|
||||
});
|
||||
|
||||
await knex.raw('CREATE UNIQUE INDEX media_sfw_id ON media_sfw(id)');
|
||||
|
||||
await knex.raw(`
|
||||
CREATE FUNCTION get_random_sfw_media_id() RETURNS varchar AS $$
|
||||
SELECT id FROM media
|
||||
WHERE is_sfw = true
|
||||
SELECT id FROM media_sfw
|
||||
ORDER BY random()
|
||||
LIMIT 1;
|
||||
$$ LANGUAGE sql STABLE;
|
||||
@@ -143,6 +149,36 @@ exports.up = async (knex) => {
|
||||
.defaultTo(knex.raw('get_random_sfw_media_id()'));
|
||||
});
|
||||
|
||||
await knex.schema.createTable('media_biometrics', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
table.string('media_id', 21)
|
||||
.references('id')
|
||||
.inTable('media')
|
||||
.notNullable()
|
||||
.onDelete('cascade');
|
||||
|
||||
table.integer('width');
|
||||
table.integer('height');
|
||||
|
||||
table.integer('face_index')
|
||||
.notNullable()
|
||||
.defaultTo(0);
|
||||
|
||||
table.json('biometrics');
|
||||
table.specificType('embedding', 'vector(1024)');
|
||||
|
||||
table.datetime('updated_at')
|
||||
.notNullable()
|
||||
.defaultTo(knex.fn.now());
|
||||
|
||||
table.datetime('created_at')
|
||||
.notNullable()
|
||||
.defaultTo(knex.fn.now());
|
||||
|
||||
table.unique(['media_id', 'face_index']);
|
||||
});
|
||||
|
||||
await knex.schema.createTable('tags_groups', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
@@ -335,7 +371,7 @@ exports.up = async (knex) => {
|
||||
|
||||
table.decimal('shoe_size');
|
||||
table.integer('leg');
|
||||
table.integer('foot');
|
||||
table.decimal('foot');
|
||||
table.integer('thigh');
|
||||
|
||||
table.string('blood_type');
|
||||
@@ -357,6 +393,9 @@ exports.up = async (knex) => {
|
||||
|
||||
table.string('agency');
|
||||
|
||||
table.boolean('allow_global_match');
|
||||
table.text('comment');
|
||||
|
||||
table.text('avatar_media_id', 21)
|
||||
.references('id')
|
||||
.inTable('media');
|
||||
@@ -443,7 +482,7 @@ exports.up = async (knex) => {
|
||||
|
||||
table.decimal('shoe_size');
|
||||
table.integer('leg');
|
||||
table.integer('foot');
|
||||
table.decimal('foot');
|
||||
table.integer('thigh');
|
||||
|
||||
table.string('blood_type');
|
||||
@@ -655,6 +694,7 @@ exports.up = async (knex) => {
|
||||
.inTable('actors');
|
||||
|
||||
table.integer('profile_id', 12)
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('actors_profiles')
|
||||
.onDelete('cascade');
|
||||
@@ -672,8 +712,6 @@ exports.up = async (knex) => {
|
||||
table.unique(['profile_id', 'media_id']);
|
||||
});
|
||||
|
||||
await knex.raw('CREATE UNIQUE INDEX unique_main_avatars ON actors_avatars (actor_id) WHERE (profile_id IS NULL);');
|
||||
|
||||
await knex.schema.createTable('actors_photos', (table) => {
|
||||
table.integer('actor_id', 12)
|
||||
.notNullable()
|
||||
@@ -713,12 +751,20 @@ exports.up = async (knex) => {
|
||||
table.datetime('pinged_at');
|
||||
table.datetime('verified_at');
|
||||
|
||||
table.unique(['actor_id', 'platform', 'handle']);
|
||||
table.unique(['actor_id', 'url']);
|
||||
});
|
||||
|
||||
await knex.raw('ALTER TABLE actors_socials ADD CONSTRAINT socials_url_or_handle CHECK (num_nulls(handle, url) = 1);');
|
||||
await knex.raw('ALTER TABLE actors_socials ADD CONSTRAINT socials_handle_and_platform CHECK (num_nulls(platform, handle) = 2 or num_nulls(platform, handle) = 0);');
|
||||
await knex.raw('CREATE UNIQUE INDEX actors_socials_actor_id_platform_handle_lower_unique ON actors_socials (actor_id, platform, lower(handle));');
|
||||
|
||||
await knex.schema.createTable('languages', (table) => {
|
||||
table.string('alpha2')
|
||||
.primary();
|
||||
|
||||
table.text('name');
|
||||
table.text('name_native');
|
||||
});
|
||||
|
||||
await knex.schema.createTable('releases', (table) => {
|
||||
table.increments('id');
|
||||
@@ -741,6 +787,7 @@ exports.up = async (knex) => {
|
||||
table.text('url', 1000);
|
||||
table.text('title');
|
||||
table.specificType('alt_titles', 'text ARRAY');
|
||||
table.specificType('alt_descriptions', 'text ARRAY');
|
||||
table.text('slug');
|
||||
|
||||
table.timestamp('date');
|
||||
@@ -758,6 +805,13 @@ exports.up = async (knex) => {
|
||||
table.enum('date_precision', ['year', 'month', 'week', 'day', 'hour', 'minute', 'second'])
|
||||
.defaultTo('day');
|
||||
|
||||
table.enum('production_date_precision', ['year', 'month', 'week', 'day', 'hour', 'minute', 'second'])
|
||||
.defaultTo('day');
|
||||
|
||||
table.string('language_alpha2')
|
||||
.references('alpha2')
|
||||
.inTable('languages');
|
||||
|
||||
table.text('description');
|
||||
|
||||
table.integer('duration')
|
||||
@@ -773,6 +827,8 @@ exports.up = async (knex) => {
|
||||
|
||||
table.text('comment');
|
||||
|
||||
table.json('attributes');
|
||||
|
||||
table.integer('created_batch_id', 12)
|
||||
.references('id')
|
||||
.inTable('batches')
|
||||
@@ -937,6 +993,8 @@ exports.up = async (knex) => {
|
||||
});
|
||||
|
||||
await knex.schema.createTable('releases_tags', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
table.integer('tag_id', 12)
|
||||
.references('id')
|
||||
.inTable('tags');
|
||||
@@ -947,9 +1005,13 @@ exports.up = async (knex) => {
|
||||
.inTable('releases')
|
||||
.onDelete('cascade');
|
||||
|
||||
table.integer('actor_id')
|
||||
.references('id')
|
||||
.inTable('actors');
|
||||
|
||||
table.text('original_tag');
|
||||
|
||||
table.unique(['tag_id', 'release_id']);
|
||||
table.unique(['release_id', 'original_tag']);
|
||||
table.index('tag_id');
|
||||
table.index('release_id');
|
||||
|
||||
@@ -958,12 +1020,7 @@ exports.up = async (knex) => {
|
||||
.defaultTo('scraper');
|
||||
});
|
||||
|
||||
await knex.schema.createTable('releases_search', (table) => {
|
||||
table.integer('release_id', 16)
|
||||
.references('id')
|
||||
.inTable('releases')
|
||||
.onDelete('cascade');
|
||||
});
|
||||
await knex.raw('CREATE UNIQUE INDEX releases_tags_tag_id_release_id_actor_id ON releases_tags (tag_id, release_id, COALESCE(actor_id, -1));');
|
||||
|
||||
await knex.schema.createTable('movies', (table) => {
|
||||
table.increments('id');
|
||||
@@ -984,6 +1041,7 @@ exports.up = async (knex) => {
|
||||
table.text('url', 1000);
|
||||
table.text('title');
|
||||
table.specificType('alt_titles', 'text ARRAY');
|
||||
table.specificType('alt_descriptions', 'text ARRAY');
|
||||
table.text('slug');
|
||||
|
||||
table.timestamp('date');
|
||||
@@ -1000,6 +1058,8 @@ exports.up = async (knex) => {
|
||||
|
||||
table.text('comment');
|
||||
|
||||
table.json('attributes');
|
||||
|
||||
table.integer('created_batch_id', 12)
|
||||
.references('id')
|
||||
.inTable('batches')
|
||||
@@ -1134,13 +1194,6 @@ exports.up = async (knex) => {
|
||||
table.unique(['movie_id', 'media_id']);
|
||||
});
|
||||
|
||||
await knex.schema.createTable('movies_search', (table) => {
|
||||
table.integer('movie_id', 16)
|
||||
.references('id')
|
||||
.inTable('movies')
|
||||
.onDelete('cascade');
|
||||
});
|
||||
|
||||
await knex.schema.createTable('series', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
@@ -1159,6 +1212,7 @@ exports.up = async (knex) => {
|
||||
table.text('url', 1000);
|
||||
table.text('title');
|
||||
table.specificType('alt_titles', 'text ARRAY');
|
||||
table.specificType('alt_descriptions', 'text ARRAY');
|
||||
table.text('slug');
|
||||
|
||||
table.timestamp('date');
|
||||
@@ -1175,6 +1229,8 @@ exports.up = async (knex) => {
|
||||
|
||||
table.text('comment');
|
||||
|
||||
table.json('attributes');
|
||||
|
||||
table.integer('created_batch_id', 12)
|
||||
.references('id')
|
||||
.inTable('batches')
|
||||
@@ -1288,13 +1344,6 @@ exports.up = async (knex) => {
|
||||
table.unique(['serie_id', 'media_id']);
|
||||
});
|
||||
|
||||
await knex.schema.createTable('series_search', (table) => {
|
||||
table.integer('serie_id', 16)
|
||||
.references('id')
|
||||
.inTable('series')
|
||||
.onDelete('cascade');
|
||||
});
|
||||
|
||||
await knex.schema.createTable('chapters', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
@@ -1316,6 +1365,8 @@ exports.up = async (knex) => {
|
||||
table.text('title');
|
||||
table.text('description');
|
||||
|
||||
table.datetime('date');
|
||||
|
||||
table.integer('created_batch_id', 12)
|
||||
.references('id')
|
||||
.inTable('batches')
|
||||
@@ -1379,6 +1430,34 @@ exports.up = async (knex) => {
|
||||
table.unique(['tag_id', 'chapter_id']);
|
||||
});
|
||||
|
||||
await knex.schema.createTable('chapters_trailers', (table) => {
|
||||
table.integer('chapter_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('chapters')
|
||||
.onDelete('cascade');
|
||||
|
||||
table.text('media_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('media')
|
||||
.onDelete('cascade');
|
||||
});
|
||||
|
||||
await knex.schema.createTable('chapters_teasers', (table) => {
|
||||
table.integer('chapter_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('chapters')
|
||||
.onDelete('cascade');
|
||||
|
||||
table.text('media_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('media')
|
||||
.onDelete('cascade');
|
||||
});
|
||||
|
||||
await knex.schema.createTable('users_roles', (table) => {
|
||||
table.string('role')
|
||||
.primary();
|
||||
@@ -1396,7 +1475,9 @@ exports.up = async (knex) => {
|
||||
{ subject: 'actor', action: 'create' },
|
||||
{ subject: 'actor', action: 'update' },
|
||||
{ subject: 'actor', action: 'delete' },
|
||||
{ plainUrls: true },
|
||||
{ subject: 'actor', action: 'merge' },
|
||||
{ subject: 'sync' },
|
||||
{ subject: 'plainUrls' },
|
||||
]),
|
||||
},
|
||||
{
|
||||
@@ -1442,6 +1523,8 @@ exports.up = async (knex) => {
|
||||
|
||||
table.specificType('last_ip', 'cidr');
|
||||
|
||||
table.jsonb('settings');
|
||||
|
||||
table.datetime('created_at')
|
||||
.notNullable()
|
||||
.defaultTo(knex.fn.now());
|
||||
@@ -1660,6 +1743,98 @@ exports.up = async (knex) => {
|
||||
.defaultTo(knex.fn.now());
|
||||
});
|
||||
|
||||
await knex.schema.createTable('feeds', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
table.integer('user_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('users')
|
||||
.onDelete('cascade');
|
||||
|
||||
table.string('name')
|
||||
.notNullable();
|
||||
|
||||
table.string('slug')
|
||||
.notNullable();
|
||||
|
||||
table.boolean('public');
|
||||
table.boolean('primary');
|
||||
|
||||
table.text('comment');
|
||||
table.json('meta');
|
||||
|
||||
table.datetime('created_at')
|
||||
.notNullable()
|
||||
.defaultTo(knex.fn.now());
|
||||
});
|
||||
|
||||
await knex.schema.createTable('feeds_entities', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
table.integer('feed_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('feeds')
|
||||
.onDelete('cascade');
|
||||
|
||||
table.integer('entity_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('entities')
|
||||
.onDelete('cascade');
|
||||
|
||||
table.text('comment');
|
||||
|
||||
table.datetime('created_at')
|
||||
.notNullable()
|
||||
.defaultTo(knex.fn.now());
|
||||
});
|
||||
|
||||
await knex.schema.createTable('feeds_actors', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
table.integer('feed_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('feeds')
|
||||
.onDelete('cascade');
|
||||
|
||||
table.integer('actor_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('actors')
|
||||
.onDelete('cascade');
|
||||
|
||||
table.text('comment');
|
||||
|
||||
table.datetime('created_at')
|
||||
.notNullable()
|
||||
.defaultTo(knex.fn.now());
|
||||
});
|
||||
|
||||
await knex.schema.createTable('feeds_tags', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
table.integer('feed_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('feeds')
|
||||
.onDelete('cascade');
|
||||
|
||||
table.integer('tag_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('tags')
|
||||
.onDelete('cascade');
|
||||
|
||||
table.text('comment');
|
||||
|
||||
table.datetime('created_at')
|
||||
.notNullable()
|
||||
.defaultTo(knex.fn.now());
|
||||
});
|
||||
|
||||
await knex.schema.createTable('alerts', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
@@ -1698,6 +1873,10 @@ exports.up = async (knex) => {
|
||||
.notNullable()
|
||||
.defaultTo(false);
|
||||
|
||||
table.boolean('is_expired')
|
||||
.notNullable()
|
||||
.defaultTo(false);
|
||||
|
||||
table.json('meta');
|
||||
table.text('comment');
|
||||
|
||||
@@ -1878,6 +2057,31 @@ exports.up = async (knex) => {
|
||||
);
|
||||
});
|
||||
|
||||
await knex.schema.createMaterializedView('alerts_users_scenes', (view) => {
|
||||
view.columns('user_id', 'scene_id', 'alert_ids');
|
||||
|
||||
view.as(
|
||||
knex('alerts_scenes')
|
||||
.select(
|
||||
'alerts.user_id',
|
||||
'alerts_scenes.scene_id',
|
||||
knex.raw('array_agg(distinct alerts.id) as alert_ids'),
|
||||
knex.raw('(alerts_tags.id is null and alerts_entities.id is null and alerts_matches.id is null and related_scenes.id is null) as is_only'),
|
||||
)
|
||||
.leftJoin('alerts', 'alerts.id', 'alerts_scenes.alert_id')
|
||||
.leftJoin('alerts_entities', 'alerts_entities.alert_id', 'alerts_scenes.alert_id')
|
||||
.leftJoin('alerts_tags', 'alerts_tags.alert_id', 'alerts_scenes.alert_id')
|
||||
.leftJoin('alerts_matches', 'alerts_matches.alert_id', 'alerts_scenes.alert_id')
|
||||
.leftJoin('alerts_scenes as related_scenes', (joinBuilder) => {
|
||||
joinBuilder
|
||||
.on('related_scenes.alert_id', 'alerts_scenes.alert_id')
|
||||
.on('related_scenes.scene_id', '!=', 'alerts_scenes.scene_id');
|
||||
})
|
||||
.where('alerts.is_expired', false)
|
||||
.groupBy(['user_id', 'alerts_scenes.scene_id', 'is_only']),
|
||||
);
|
||||
});
|
||||
|
||||
await knex.schema.createTable('notifications', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
@@ -1901,6 +2105,18 @@ exports.up = async (knex) => {
|
||||
.notNullable()
|
||||
.defaultTo(false);
|
||||
|
||||
table.datetime('seen_at');
|
||||
|
||||
table.boolean('notify')
|
||||
.notNullable()
|
||||
.defaultTo(true);
|
||||
|
||||
table.boolean('email')
|
||||
.notNullable()
|
||||
.defaultTo(false);
|
||||
|
||||
table.datetime('emailed_at');
|
||||
|
||||
table.datetime('created_at')
|
||||
.notNullable()
|
||||
.defaultTo(knex.fn.now());
|
||||
@@ -2177,12 +2393,20 @@ exports.up = async (knex) => {
|
||||
.inTable('entities');
|
||||
});
|
||||
|
||||
await knex.schema.createTable('sync', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
table.string('domain');
|
||||
table.specificType('item_ids', 'integer array');
|
||||
|
||||
table.text('comment');
|
||||
|
||||
table.datetime('created_at')
|
||||
.defaultTo(knex.fn.now());
|
||||
});
|
||||
|
||||
// SEARCH AND SORT
|
||||
await knex.raw(`
|
||||
ALTER TABLE releases_search ADD COLUMN document tsvector;
|
||||
ALTER TABLE movies_search ADD COLUMN document tsvector;
|
||||
ALTER TABLE series_search ADD COLUMN document tsvector;
|
||||
|
||||
/* allow scenes without dates to be mixed inbetween scenes with dates */
|
||||
ALTER TABLE releases
|
||||
ADD COLUMN effective_date timestamptz
|
||||
@@ -2200,19 +2424,12 @@ exports.up = async (knex) => {
|
||||
// INDEXES
|
||||
await knex.raw(`
|
||||
CREATE UNIQUE INDEX unique_actor_slugs_network ON actors (slug, entity_id, entry_id);
|
||||
CREATE UNIQUE INDEX unique_actor_slugs ON actors (slug) WHERE entity_id IS NULL;
|
||||
CREATE UNIQUE INDEX unique_actor_slugs ON actors (slug, entry_id) WHERE entity_id IS NULL;
|
||||
|
||||
CREATE UNIQUE INDEX unique_entity_campaigns_banner_url ON campaigns (entity_id, url, banner_id) WHERE affiliate_id IS NULL;
|
||||
CREATE UNIQUE INDEX unique_entity_campaigns_url ON campaigns (entity_id, url) WHERE banner_id IS NULL AND affiliate_id IS NULL;
|
||||
CREATE UNIQUE INDEX unique_entity_campaigns_banner_affiliate ON campaigns (entity_id, affiliate_id, banner_id) WHERE url IS NULL;
|
||||
CREATE UNIQUE INDEX unique_entity_campaigns_affiliate ON campaigns (entity_id, affiliate_id) WHERE banner_id IS NULL AND url IS NULL;
|
||||
|
||||
CREATE UNIQUE INDEX releases_search_unique ON releases_search (release_id);
|
||||
CREATE UNIQUE INDEX movies_search_unique ON movies_search (movie_id);
|
||||
CREATE INDEX releases_search_index ON releases_search USING GIN (document);
|
||||
CREATE INDEX movies_search_index ON movies_search USING GIN (document);
|
||||
CREATE UNIQUE INDEX series_search_unique ON series_search (serie_id);
|
||||
CREATE INDEX series_search_index ON series_search USING GIN (document);
|
||||
`);
|
||||
|
||||
// FUNCTIONS
|
||||
@@ -2275,7 +2492,8 @@ exports.up = async (knex) => {
|
||||
stashes.id as stash_id,
|
||||
COUNT(DISTINCT stashes_scenes)::integer as stashed_scenes,
|
||||
COUNT(DISTINCT stashes_movies)::integer as stashed_movies,
|
||||
COUNT(DISTINCT stashes_actors)::integer as stashed_actors
|
||||
COUNT(DISTINCT stashes_actors)::integer as stashed_actors,
|
||||
MAX(GREATEST(stashes_scenes.created_at, stashes_movies.created_at, stashes_actors.created_at)) as last_stash_at
|
||||
FROM stashes
|
||||
LEFT JOIN stashes_scenes ON stashes_scenes.stash_id = stashes.id
|
||||
LEFT JOIN stashes_movies ON stashes_movies.stash_id = stashes.id
|
||||
@@ -2372,6 +2590,8 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
|
||||
DROP TABLE IF EXISTS chapters_tags CASCADE;
|
||||
DROP TABLE IF EXISTS chapters_posters CASCADE;
|
||||
DROP TABLE IF EXISTS chapters_photos CASCADE;
|
||||
DROP TABLE IF EXISTS chapters_trailers CASCADE;
|
||||
DROP TABLE IF EXISTS chapters_teasers CASCADE;
|
||||
|
||||
DROP TABLE IF EXISTS banners_tags CASCADE;
|
||||
DROP TABLE IF EXISTS banners CASCADE;
|
||||
@@ -2381,6 +2601,7 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
|
||||
|
||||
DROP TABLE IF EXISTS batches CASCADE;
|
||||
DROP TABLE IF EXISTS fingerprints_types CASCADE;
|
||||
DROP TABLE IF EXISTS sync CASCADE;
|
||||
|
||||
DROP TABLE IF EXISTS actors_avatars CASCADE;
|
||||
DROP TABLE IF EXISTS actors_photos CASCADE;
|
||||
@@ -2412,8 +2633,10 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
|
||||
DROP TABLE IF EXISTS sites CASCADE;
|
||||
DROP TABLE IF EXISTS studios CASCADE;
|
||||
DROP TABLE IF EXISTS media_sfw CASCADE;
|
||||
DROP TABLE IF EXISTS media_biometrics CASCADE;
|
||||
DROP TABLE IF EXISTS media CASCADE;
|
||||
DROP TABLE IF EXISTS countries CASCADE;
|
||||
DROP TABLE IF EXISTS languages CASCADE;
|
||||
DROP TABLE IF EXISTS networks CASCADE;
|
||||
|
||||
DROP TABLE IF EXISTS entities_types CASCADE;
|
||||
@@ -2426,6 +2649,11 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
|
||||
DROP TABLE IF EXISTS stashes_actors CASCADE;
|
||||
DROP TABLE IF EXISTS stashes CASCADE;
|
||||
|
||||
DROP TABLE IF EXISTS feeds_entities CASCADE;
|
||||
DROP TABLE IF EXISTS feeds_actors CASCADE;
|
||||
DROP TABLE IF EXISTS feeds_tags CASCADE;
|
||||
DROP TABLE IF EXISTS feeds CASCADE;
|
||||
|
||||
DROP TABLE IF EXISTS alerts_scenes CASCADE;
|
||||
DROP TABLE IF EXISTS alerts_actors CASCADE;
|
||||
DROP TABLE IF EXISTS alerts_tags CASCADE;
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
exports.up = async (knex) => {
|
||||
await knex.schema.alterTable('releases', (table) => {
|
||||
table.json('attributes');
|
||||
});
|
||||
|
||||
await knex.schema.alterTable('movies', (table) => {
|
||||
table.json('attributes');
|
||||
});
|
||||
};
|
||||
|
||||
exports.down = async (knex) => {
|
||||
await knex.schema.alterTable('releases', (table) => {
|
||||
table.dropColumn('attributes');
|
||||
});
|
||||
|
||||
await knex.schema.alterTable('movies', (table) => {
|
||||
table.dropColumn('attributes');
|
||||
});
|
||||
};
|
||||
@@ -1,11 +0,0 @@
|
||||
exports.up = async function(knex) {
|
||||
await knex.schema.alterTable('releases', (table) => {
|
||||
table.specificType('alt_descriptions', 'text ARRAY');
|
||||
});
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.schema.alterTable('releases', (table) => {
|
||||
table.dropColumn('alt_descriptions');
|
||||
});
|
||||
};
|
||||
@@ -1,58 +0,0 @@
|
||||
exports.up = async function(knex) {
|
||||
await knex.schema.alterTable('chapters', (table) => {
|
||||
table.datetime('date');
|
||||
});
|
||||
|
||||
await knex.schema.createTable('chapters_trailers', (table) => {
|
||||
table.integer('chapter_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('chapters')
|
||||
.onDelete('cascade');
|
||||
|
||||
table.text('media_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('media')
|
||||
.onDelete('cascade');
|
||||
});
|
||||
|
||||
await knex.schema.createTable('chapters_teasers', (table) => {
|
||||
table.integer('chapter_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('chapters')
|
||||
.onDelete('cascade');
|
||||
|
||||
table.text('media_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('media')
|
||||
.onDelete('cascade');
|
||||
});
|
||||
|
||||
await knex.schema.alterTable('entities', (table) => {
|
||||
table.string('name_stylized');
|
||||
});
|
||||
|
||||
await knex.schema.alterTable('movies', (table) => {
|
||||
table.specificType('alt_descriptions', 'text ARRAY');
|
||||
});
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.schema.alterTable('chapters', (table) => {
|
||||
table.dropColumn('date');
|
||||
});
|
||||
|
||||
await knex.schema.alterTable('entities', (table) => {
|
||||
table.dropColumn('name_stylized');
|
||||
});
|
||||
|
||||
await knex.schema.dropTable('chapters_trailers');
|
||||
await knex.schema.dropTable('chapters_teasers');
|
||||
|
||||
await knex.schema.alterTable('movies', (table) => {
|
||||
table.dropColumn('alt_descriptions');
|
||||
});
|
||||
};
|
||||
@@ -1,28 +0,0 @@
|
||||
exports.up = async function(knex) {
|
||||
await knex.schema.createMaterializedView('media_sfw', (view) => {
|
||||
view.as(knex('media').select('id').where('is_sfw', true));
|
||||
});
|
||||
|
||||
await knex.raw('CREATE UNIQUE INDEX media_sfw_id ON media_sfw(id)');
|
||||
|
||||
await knex.raw(`
|
||||
CREATE OR REPLACE FUNCTION get_random_sfw_media_id() RETURNS varchar AS $$
|
||||
SELECT id FROM media_sfw
|
||||
ORDER BY random()
|
||||
LIMIT 1;
|
||||
$$ LANGUAGE sql STABLE;
|
||||
`);
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.raw(`
|
||||
CREATE OR REPLACE FUNCTION get_random_sfw_media_id() RETURNS varchar AS $$
|
||||
SELECT id FROM media
|
||||
WHERE is_sfw = true
|
||||
ORDER BY random()
|
||||
LIMIT 1;
|
||||
$$ LANGUAGE sql STABLE;
|
||||
`);
|
||||
|
||||
await knex.schema.dropMaterializedView('media_sfw');
|
||||
};
|
||||
@@ -1,31 +0,0 @@
|
||||
exports.up = async (knex) => {
|
||||
// dedupe
|
||||
await knex.raw(`
|
||||
DELETE
|
||||
FROM releases_tags
|
||||
WHERE ctid IN
|
||||
(
|
||||
SELECT ctid
|
||||
FROM(
|
||||
SELECT
|
||||
*,
|
||||
ctid,
|
||||
row_number() OVER (PARTITION BY release_id, original_tag ORDER BY ctid)
|
||||
FROM releases_tags
|
||||
)s
|
||||
WHERE row_number >= 2
|
||||
)
|
||||
`);
|
||||
|
||||
await knex.schema.alterTable('releases_tags', (table) => {
|
||||
table.increments('id');
|
||||
table.unique(['release_id', 'original_tag']);
|
||||
});
|
||||
};
|
||||
|
||||
exports.down = async (knex) => {
|
||||
await knex.schema.alterTable('releases_tags', (table) => {
|
||||
table.dropColumn('id');
|
||||
table.dropUnique(['release_id', 'original_tag']);
|
||||
});
|
||||
};
|
||||
@@ -1,23 +0,0 @@
|
||||
exports.up = async function(knex) {
|
||||
await knex.schema.alterTable('actors', (table) => {
|
||||
table.decimal('foot')
|
||||
.alter();
|
||||
});
|
||||
|
||||
await knex.schema.alterTable('actors_profiles', (table) => {
|
||||
table.decimal('foot')
|
||||
.alter();
|
||||
});
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.schema.alterTable('actors', (table) => {
|
||||
table.integer('foot')
|
||||
.alter();
|
||||
});
|
||||
|
||||
await knex.schema.alterTable('actors_profiles', (table) => {
|
||||
table.integer('foot')
|
||||
.alter();
|
||||
});
|
||||
};
|
||||
@@ -1,13 +0,0 @@
|
||||
exports.up = async function(knex) {
|
||||
await knex.schema.alterTable('series', (table) => {
|
||||
table.specificType('alt_descriptions', 'text ARRAY');
|
||||
table.json('attributes');
|
||||
});
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.schema.alterTable('series', (table) => {
|
||||
table.dropColumn('alt_descriptions');
|
||||
table.dropColumn('attributes');
|
||||
});
|
||||
};
|
||||
@@ -1,21 +0,0 @@
|
||||
exports.up = async function(knex) {
|
||||
await knex.schema.alterTable('releases_tags', (table) => {
|
||||
table.integer('actor_id')
|
||||
.references('id')
|
||||
.inTable('actors');
|
||||
|
||||
table.dropUnique(['tag_id', 'release_id']);
|
||||
});
|
||||
|
||||
await knex.raw('CREATE UNIQUE INDEX releases_tags_tag_id_release_id_actor_id ON releases_tags (tag_id, release_id, COALESCE(actor_id, -1))');
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.schema.alterTable('releases_tags', (table) => {
|
||||
table.dropColumn('actor_id');
|
||||
|
||||
table.unique(['tag_id', 'release_id']);
|
||||
});
|
||||
|
||||
await knex.raw('DROP INDEX IF EXISTS releases_tags_tag_id_release_id_actor_id');
|
||||
};
|
||||
@@ -1,27 +0,0 @@
|
||||
exports.up = async function(knex) {
|
||||
await knex.schema.createTable('languages', (table) => {
|
||||
table.string('alpha2')
|
||||
.primary();
|
||||
|
||||
table.text('name');
|
||||
table.text('name_native');
|
||||
});
|
||||
|
||||
await knex.schema.alterTable('releases', (table) => {
|
||||
table.enum('production_date_precision', ['year', 'month', 'week', 'day', 'hour', 'minute', 'second'])
|
||||
.defaultTo('day');
|
||||
|
||||
table.string('language_alpha2')
|
||||
.references('alpha2')
|
||||
.inTable('languages');
|
||||
});
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.schema.alterTable('releases', (table) => {
|
||||
table.dropColumn('production_date_precision');
|
||||
table.dropColumn('language_alpha2');
|
||||
});
|
||||
|
||||
await knex.schema.dropTable('languages');
|
||||
};
|
||||
@@ -1,100 +0,0 @@
|
||||
exports.up = async function(knex) {
|
||||
await knex.schema.createTable('feeds', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
table.integer('user_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('users')
|
||||
.onDelete('cascade');
|
||||
|
||||
table.string('name')
|
||||
.notNullable();
|
||||
|
||||
table.string('slug')
|
||||
.notNullable();
|
||||
|
||||
table.boolean('public');
|
||||
table.boolean('primary');
|
||||
|
||||
table.text('comment');
|
||||
table.json('meta');
|
||||
|
||||
table.datetime('created_at')
|
||||
.notNullable()
|
||||
.defaultTo(knex.fn.now());
|
||||
});
|
||||
|
||||
await knex.schema.createTable('feeds_entities', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
table.integer('feed_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('feeds')
|
||||
.onDelete('cascade');
|
||||
|
||||
table.integer('entity_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('entities')
|
||||
.onDelete('cascade');
|
||||
|
||||
table.text('comment');
|
||||
|
||||
table.datetime('created_at')
|
||||
.notNullable()
|
||||
.defaultTo(knex.fn.now());
|
||||
});
|
||||
|
||||
await knex.schema.createTable('feeds_actors', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
table.integer('feed_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('feeds')
|
||||
.onDelete('cascade');
|
||||
|
||||
table.integer('actor_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('actors')
|
||||
.onDelete('cascade');
|
||||
|
||||
table.text('comment');
|
||||
|
||||
table.datetime('created_at')
|
||||
.notNullable()
|
||||
.defaultTo(knex.fn.now());
|
||||
});
|
||||
|
||||
await knex.schema.createTable('feeds_tags', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
table.integer('feed_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('feeds')
|
||||
.onDelete('cascade');
|
||||
|
||||
table.integer('tag_id')
|
||||
.notNullable()
|
||||
.references('id')
|
||||
.inTable('tags')
|
||||
.onDelete('cascade');
|
||||
|
||||
table.text('comment');
|
||||
|
||||
table.datetime('created_at')
|
||||
.notNullable()
|
||||
.defaultTo(knex.fn.now());
|
||||
});
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.schema.dropTable('feeds_tags');
|
||||
await knex.schema.dropTable('feeds_actors');
|
||||
await knex.schema.dropTable('feeds_entities');
|
||||
await knex.schema.dropTable('feeds');
|
||||
};
|
||||
@@ -1,59 +0,0 @@
|
||||
exports.up = async function(knex) {
|
||||
await knex.raw(`
|
||||
DROP INDEX unique_actor_slugs;
|
||||
CREATE UNIQUE INDEX unique_actor_slugs ON actors (slug, entry_id) WHERE entity_id IS NULL;
|
||||
`);
|
||||
|
||||
await knex.schema.alterTable('actors', (table) => {
|
||||
table.boolean('allow_global_match');
|
||||
table.text('comment');
|
||||
});
|
||||
|
||||
await knex('users_roles')
|
||||
.update('abilities', JSON.stringify([
|
||||
{ subject: 'scene', action: 'create' },
|
||||
{ subject: 'scene', action: 'update' },
|
||||
{ subject: 'scene', action: 'delete' },
|
||||
{ subject: 'actor', action: 'create' },
|
||||
{ subject: 'actor', action: 'update' },
|
||||
{ subject: 'actor', action: 'delete' },
|
||||
{ subject: 'actor', action: 'merge' },
|
||||
{ plainUrls: true },
|
||||
]))
|
||||
.where('role', 'admin');
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
const dupes = await knex('actors')
|
||||
.select('name', 'slug', knex.raw('count(*) as count'))
|
||||
.whereNull('entity_id')
|
||||
.groupBy('name', 'slug')
|
||||
.havingRaw('count(*) > 1')
|
||||
.orderBy('count', 'desc');
|
||||
|
||||
if (dupes.length > 0) {
|
||||
console.log('DUPES\n', dupes.map((actor) => `${actor.name} ${actor.slug} ${actor.count}`).join('\n'));
|
||||
}
|
||||
|
||||
await knex.raw(`
|
||||
DROP INDEX unique_actor_slugs;
|
||||
CREATE UNIQUE INDEX unique_actor_slugs ON actors (slug) WHERE entity_id IS NULL;
|
||||
`);
|
||||
|
||||
await knex.schema.alterTable('actors', (table) => {
|
||||
table.dropColumn('allow_global_match');
|
||||
table.dropColumn('comment');
|
||||
});
|
||||
|
||||
await knex('users_roles')
|
||||
.update('abilities', JSON.stringify([
|
||||
{ subject: 'scene', action: 'create' },
|
||||
{ subject: 'scene', action: 'update' },
|
||||
{ subject: 'scene', action: 'delete' },
|
||||
{ subject: 'actor', action: 'create' },
|
||||
{ subject: 'actor', action: 'update' },
|
||||
{ subject: 'actor', action: 'delete' },
|
||||
{ plainUrls: true },
|
||||
]))
|
||||
.where('role', 'admin');
|
||||
};
|
||||
@@ -1,87 +0,0 @@
|
||||
exports.up = async function(knex) {
|
||||
await knex.schema.createTable('sync', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
table.string('domain');
|
||||
table.specificType('item_ids', 'integer array');
|
||||
|
||||
table.text('comment');
|
||||
|
||||
table.datetime('created_at')
|
||||
.defaultTo(knex.fn.now());
|
||||
});
|
||||
|
||||
await knex('users_roles')
|
||||
.update('abilities', JSON.stringify([
|
||||
{ subject: 'scene', action: 'create' },
|
||||
{ subject: 'scene', action: 'update' },
|
||||
{ subject: 'scene', action: 'delete' },
|
||||
{ subject: 'actor', action: 'create' },
|
||||
{ subject: 'actor', action: 'update' },
|
||||
{ subject: 'actor', action: 'delete' },
|
||||
{ subject: 'actor', action: 'merge' },
|
||||
{ subject: 'sync' },
|
||||
{ subject: 'plainUrls' },
|
||||
]))
|
||||
.where('role', 'admin');
|
||||
|
||||
await knex.raw(`
|
||||
DROP TABLE IF EXISTS releases_search CASCADE;
|
||||
DROP TABLE IF EXISTS movies_search CASCADE;
|
||||
DROP TABLE IF EXISTS series_search CASCADE;
|
||||
|
||||
DROP TABLE IF EXISTS releases_search_results CASCADE;
|
||||
DROP TABLE IF EXISTS movies_search_results CASCADE;
|
||||
`);
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.schema.dropTable('sync');
|
||||
|
||||
await knex('users_roles')
|
||||
.update('abilities', JSON.stringify([
|
||||
{ subject: 'scene', action: 'create' },
|
||||
{ subject: 'scene', action: 'update' },
|
||||
{ subject: 'scene', action: 'delete' },
|
||||
{ subject: 'actor', action: 'create' },
|
||||
{ subject: 'actor', action: 'update' },
|
||||
{ subject: 'actor', action: 'delete' },
|
||||
{ subject: 'actor', action: 'merge' },
|
||||
{ plainUrls: true },
|
||||
]))
|
||||
.where('role', 'admin');
|
||||
|
||||
await knex.schema.createTable('releases_search', (table) => {
|
||||
table.integer('release_id', 16)
|
||||
.references('id')
|
||||
.inTable('releases')
|
||||
.onDelete('cascade');
|
||||
});
|
||||
|
||||
await knex.schema.createTable('movies_search', (table) => {
|
||||
table.integer('movie_id', 16)
|
||||
.references('id')
|
||||
.inTable('movies')
|
||||
.onDelete('cascade');
|
||||
});
|
||||
|
||||
await knex.schema.createTable('series_search', (table) => {
|
||||
table.integer('serie_id', 16)
|
||||
.references('id')
|
||||
.inTable('series')
|
||||
.onDelete('cascade');
|
||||
});
|
||||
|
||||
await knex.raw(`
|
||||
ALTER TABLE releases_search ADD COLUMN document tsvector;
|
||||
ALTER TABLE movies_search ADD COLUMN document tsvector;
|
||||
ALTER TABLE series_search ADD COLUMN document tsvector;
|
||||
|
||||
CREATE UNIQUE INDEX releases_search_unique ON releases_search (release_id);
|
||||
CREATE UNIQUE INDEX movies_search_unique ON movies_search (movie_id);
|
||||
CREATE INDEX releases_search_index ON releases_search USING GIN (document);
|
||||
CREATE INDEX movies_search_index ON movies_search USING GIN (document);
|
||||
CREATE UNIQUE INDEX series_search_unique ON series_search (serie_id);
|
||||
CREATE INDEX series_search_index ON series_search USING GIN (document);
|
||||
`);
|
||||
};
|
||||
@@ -1,35 +0,0 @@
|
||||
exports.up = async function(knex) {
|
||||
await knex.schema.createTable('media_biometrics', (table) => {
|
||||
table.increments('id');
|
||||
|
||||
table.string('media_id', 21)
|
||||
.references('id')
|
||||
.inTable('media')
|
||||
.notNullable()
|
||||
.onDelete('cascade');
|
||||
|
||||
table.integer('width');
|
||||
table.integer('height');
|
||||
|
||||
table.integer('face_index')
|
||||
.notNullable()
|
||||
.defaultTo(0);
|
||||
|
||||
table.json('biometrics');
|
||||
table.specificType('embedding', 'vector(1024)');
|
||||
|
||||
table.datetime('updated_at')
|
||||
.notNullable()
|
||||
.defaultTo(knex.fn.now());
|
||||
|
||||
table.datetime('created_at')
|
||||
.notNullable()
|
||||
.defaultTo(knex.fn.now());
|
||||
|
||||
table.unique(['media_id', 'face_index']);
|
||||
});
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.schema.dropTable('media_biometrics');
|
||||
};
|
||||
@@ -1,48 +0,0 @@
|
||||
exports.up = async function(knex) {
|
||||
await knex.schema.alterTable('notifications', (table) => {
|
||||
table.datetime('seen_at');
|
||||
});
|
||||
|
||||
await knex.schema.alterTable('alerts', (table) => {
|
||||
table.boolean('is_expired')
|
||||
.notNullable()
|
||||
.defaultTo(false);
|
||||
});
|
||||
|
||||
await knex.schema.createMaterializedView('alerts_users_scenes', (view) => {
|
||||
view.columns('user_id', 'scene_id', 'alert_ids');
|
||||
|
||||
view.as(
|
||||
knex('alerts_scenes')
|
||||
.select(
|
||||
'alerts.user_id',
|
||||
'alerts_scenes.scene_id',
|
||||
knex.raw('array_agg(distinct alerts.id) as alert_ids'),
|
||||
knex.raw('(alerts_tags.id is null and alerts_entities.id is null and alerts_matches.id is null and related_scenes.id is null) as is_only'),
|
||||
)
|
||||
.leftJoin('alerts', 'alerts.id', 'alerts_scenes.alert_id')
|
||||
.leftJoin('alerts_entities', 'alerts_entities.alert_id', 'alerts_scenes.alert_id')
|
||||
.leftJoin('alerts_tags', 'alerts_tags.alert_id', 'alerts_scenes.alert_id')
|
||||
.leftJoin('alerts_matches', 'alerts_matches.alert_id', 'alerts_scenes.alert_id')
|
||||
.leftJoin('alerts_scenes as related_scenes', (joinBuilder) => {
|
||||
joinBuilder
|
||||
.on('related_scenes.alert_id', 'alerts_scenes.alert_id')
|
||||
.on('related_scenes.scene_id', '!=', 'alerts_scenes.scene_id');
|
||||
})
|
||||
.where('alerts.is_expired', false)
|
||||
.groupBy(['user_id', 'alerts_scenes.scene_id', 'is_only']),
|
||||
);
|
||||
});
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.schema.dropMaterializedView('alerts_users_scenes');
|
||||
|
||||
await knex.schema.alterTable('alerts', (table) => {
|
||||
table.dropColumn('is_expired');
|
||||
});
|
||||
|
||||
await knex.schema.alterTable('notifications', (table) => {
|
||||
table.dropColumn('seen_at');
|
||||
});
|
||||
};
|
||||
@@ -1,29 +0,0 @@
|
||||
exports.up = async function(knex) {
|
||||
await knex.schema.alterTable('users', (table) => {
|
||||
table.jsonb('settings');
|
||||
});
|
||||
|
||||
await knex.schema.alterTable('notifications', (table) => {
|
||||
table.boolean('notify')
|
||||
.notNullable()
|
||||
.defaultTo(true);
|
||||
|
||||
table.boolean('email')
|
||||
.notNullable()
|
||||
.defaultTo(false);
|
||||
|
||||
table.datetime('emailed_at');
|
||||
});
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.schema.alterTable('users', (table) => {
|
||||
table.dropColumn('settings');
|
||||
});
|
||||
|
||||
await knex.schema.alterTable('notifications', (table) => {
|
||||
table.dropColumn('notify');
|
||||
table.dropColumn('email');
|
||||
table.dropColumn('emailed_at');
|
||||
});
|
||||
};
|
||||
@@ -0,0 +1,15 @@
|
||||
exports.up = async function(knex) {
|
||||
await knex.schema.alterTable('stashes', (table) => {
|
||||
table.boolean('archived')
|
||||
.notNullable()
|
||||
.defaultTo(false);
|
||||
|
||||
table.check('NOT ("primary" AND archived)', [], 'stashes_not_primary_and_archived');
|
||||
});
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.schema.alterTable('stashes', (table) => {
|
||||
table.dropColumn('archived');
|
||||
});
|
||||
};
|
||||
@@ -0,0 +1,33 @@
|
||||
exports.up = async function(knex) {
|
||||
await knex.schema.alterTable('actors', (table) => {
|
||||
table.string('date_of_birth_precision')
|
||||
.notNullable()
|
||||
.defaultTo('day');
|
||||
|
||||
table.string('date_of_death_precision')
|
||||
.notNullable()
|
||||
.defaultTo('day');
|
||||
});
|
||||
|
||||
await knex.schema.alterTable('actors_profiles', (table) => {
|
||||
table.string('date_of_birth_precision')
|
||||
.notNullable()
|
||||
.defaultTo('day');
|
||||
|
||||
table.string('date_of_death_precision')
|
||||
.notNullable()
|
||||
.defaultTo('day');
|
||||
});
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.schema.alterTable('actors', (table) => {
|
||||
table.dropColumn('date_of_birth_precision');
|
||||
table.dropColumn('date_of_death_precision');
|
||||
});
|
||||
|
||||
await knex.schema.alterTable('actors_profiles', (table) => {
|
||||
table.dropColumn('date_of_birth_precision');
|
||||
table.dropColumn('date_of_death_precision');
|
||||
});
|
||||
};
|
||||
@@ -0,0 +1,24 @@
|
||||
exports.up = async function(knex) {
|
||||
// we want to be able to assign unidentified actors
|
||||
await knex.raw(`
|
||||
ALTER TABLE releases_actors
|
||||
ALTER COLUMN actor_id DROP NOT NULL
|
||||
`);
|
||||
|
||||
await knex.schema.alterTable('releases_actors', (table) => {
|
||||
table.increments('id');
|
||||
table.string('gender');
|
||||
});
|
||||
};
|
||||
|
||||
exports.down = async function(knex) {
|
||||
await knex.raw(`
|
||||
ALTER TABLE releases_actors
|
||||
ALTER COLUMN actor_id SET NOT NULL
|
||||
`);
|
||||
|
||||
await knex.schema.alterTable('releases_actors', (table) => {
|
||||
table.dropColumn('id');
|
||||
table.dropColumn('gender');
|
||||
});
|
||||
};
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "traxxx",
|
||||
"version": "1.255.2",
|
||||
"version": "1.256.5",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "traxxx",
|
||||
"version": "1.255.2",
|
||||
"version": "1.256.5",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.458.0",
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "traxxx",
|
||||
"version": "1.255.2",
|
||||
"version": "1.256.5",
|
||||
"description": "All the latest porn releases in one place",
|
||||
"main": "src/app.js",
|
||||
"scripts": {
|
||||
|
||||
+92
-2
@@ -2,6 +2,10 @@ const upsert = require('../src/utils/upsert');
|
||||
const slugify = require('../src/utils/slugify');
|
||||
|
||||
const groups = [
|
||||
{
|
||||
slug: 'accessories',
|
||||
name: 'Accessories',
|
||||
},
|
||||
{
|
||||
slug: 'age',
|
||||
name: 'Age',
|
||||
@@ -344,10 +348,31 @@ const tags = [
|
||||
slug: 'brunette',
|
||||
group: 'hair',
|
||||
},
|
||||
{
|
||||
name: 'body writing',
|
||||
slug: 'body-writing',
|
||||
description: 'Grab a marker or a lipstick and use your body as a canvas. Label all your holes and make it clear what they should call you.',
|
||||
},
|
||||
{
|
||||
name: 'boots',
|
||||
slug: 'boots',
|
||||
group: 'clothing',
|
||||
},
|
||||
{
|
||||
name: 'pleasers',
|
||||
slug: 'pleasers',
|
||||
group: 'clothing',
|
||||
},
|
||||
{
|
||||
name: 'bukkake',
|
||||
slug: 'bukkake',
|
||||
description: 'Getting ejaculated on the face by a group of three or more men, often following a [blowbang](/tag/blowbang) or [gangbang](/tag/gangbang).',
|
||||
description: 'Getting ejaculated on the face by a group of three or more men, often following a [blowbang](/tag/blowbang) or [gangbang](/tag/gangbang). If the mouth filled up directly or the cum is collected and drunk from a cup, it can be referred to as a [gokkun](/tag/gokkun).',
|
||||
group: 'finish',
|
||||
},
|
||||
{
|
||||
name: 'gokkun',
|
||||
slug: 'gokkun',
|
||||
description: 'Drinking cum from a group of three or more men, either ejaculated directly into the mouth or collected and drank from a cup, often following a [blowbang](/tag/blowbang) or [gangbang](/tag/gangbang). It is essentially the swallowing equivalent of a [bukkake](/tag/bukkake).',
|
||||
group: 'finish',
|
||||
},
|
||||
{
|
||||
@@ -406,6 +431,10 @@ const tags = [
|
||||
name: 'cum',
|
||||
slug: 'cum',
|
||||
},
|
||||
{
|
||||
name: 'cum covered fucking',
|
||||
slug: 'ccf',
|
||||
},
|
||||
{
|
||||
name: 'cum drunk',
|
||||
slug: 'cum-drunk',
|
||||
@@ -688,7 +717,12 @@ const tags = [
|
||||
{
|
||||
name: 'gangbang',
|
||||
slug: 'gangbang',
|
||||
description: 'Getting fucked by three or more cocks, at least two at the same time, often but not necessarily involving a [blowbang](/tag/blowbang), [spitroast](/tag/mfm), [double penetration](/tag/dp) and [airtight](/tag/airtight). If you get fucked by one guy at a time, it might be considered a [trainbang](/tag/trainbang). In a reverse gangbang, multiple women fuck one man.', /* eslint-disable-line max-len */
|
||||
description: 'Getting fucked by three or more cocks, at least two at the same time, often but not necessarily involving a [blowbang](/tag/blowbang), [spitroast](/tag/mfm), [double penetration](/tag/dp) and [airtight](/tag/airtight). If you get fucked by one guy at a time, it might be considered a [trainbang](/tag/trainbang). In a [reverse gangbang](/tag/reverse-gangbang), multiple women fuck one man.', /* eslint-disable-line max-len */
|
||||
group: 'group',
|
||||
},
|
||||
{
|
||||
name: 'reverse gangbang',
|
||||
slug: 'reverse-gangbang',
|
||||
group: 'group',
|
||||
},
|
||||
{
|
||||
@@ -1264,6 +1298,22 @@ const tags = [
|
||||
slug: 'toy-dp',
|
||||
description: 'Getting [double penetrated](/tag/dp) with dildos, strap-ons or buttplugs. You can use both ends of a [double dildo](/tag/double-dildo) for a [double dildo DP](/tag/double-dildo-dp).',
|
||||
},
|
||||
{
|
||||
name: 'toy DVP',
|
||||
slug: 'toy-dvp',
|
||||
},
|
||||
{
|
||||
name: 'toy DAP',
|
||||
slug: 'toy-dap',
|
||||
},
|
||||
{
|
||||
name: 'toy TAP',
|
||||
slug: 'toy-tap',
|
||||
},
|
||||
{
|
||||
name: 'toy TVP',
|
||||
slug: 'toy-tvp',
|
||||
},
|
||||
{
|
||||
name: 'finger DP',
|
||||
slug: 'finger-dp',
|
||||
@@ -1272,6 +1322,16 @@ const tags = [
|
||||
name: 'transsexual',
|
||||
slug: 'transsexual',
|
||||
},
|
||||
{
|
||||
name: 'trans man',
|
||||
slug: 'trans-man',
|
||||
description: 'A man who was assigned female at birth and has typically undergone a social transition, and may but not necessarily have had gender-affirming surgeries.',
|
||||
},
|
||||
{
|
||||
name: 'trans woman',
|
||||
slug: 'trans-woman',
|
||||
description: 'woman man who was assigned male at birth and has typically undergone a social transition, and may but not necessarily have had gender-affirming surgeries.',
|
||||
},
|
||||
{
|
||||
name: 'DA triple penetration',
|
||||
slug: 'da-tp',
|
||||
@@ -1465,6 +1525,16 @@ const tags = [
|
||||
name: 'pregnant',
|
||||
slug: 'pregnant',
|
||||
},
|
||||
{
|
||||
name: 'collar',
|
||||
slug: 'collar',
|
||||
group: 'accessories',
|
||||
},
|
||||
{
|
||||
name: 'choker',
|
||||
slug: 'choker',
|
||||
group: 'accessories',
|
||||
},
|
||||
];
|
||||
|
||||
const aliases = [
|
||||
@@ -1715,6 +1785,10 @@ const aliases = [
|
||||
name: 'busty - big boobs',
|
||||
for: 'big-boobs',
|
||||
},
|
||||
{
|
||||
name: 'big breasts',
|
||||
for: 'big-boobs',
|
||||
},
|
||||
{
|
||||
name: 'busty: big beautiful breast',
|
||||
for: 'big-boobs',
|
||||
@@ -3139,6 +3213,22 @@ const aliases = [
|
||||
name: 'fist',
|
||||
for: 'fisting',
|
||||
},
|
||||
{
|
||||
name: 'boot',
|
||||
for: 'boots',
|
||||
},
|
||||
{
|
||||
name: 'pleaser boots',
|
||||
for: 'pleasers',
|
||||
},
|
||||
{
|
||||
name: 'pleaser shoes',
|
||||
for: 'pleasers',
|
||||
},
|
||||
{
|
||||
name: 'pleaser heels',
|
||||
for: 'pleasers',
|
||||
},
|
||||
];
|
||||
|
||||
const priorities = [ // higher index is higher priority
|
||||
|
||||
+12
-1
@@ -172,7 +172,7 @@ const networks = [
|
||||
{
|
||||
slug: 'familystrokes',
|
||||
name: 'Family Strokes',
|
||||
url: 'https://www.familystrokes.com/',
|
||||
url: 'https://www.familystrokes.com',
|
||||
parent: 'paperstreetmedia',
|
||||
parameters: {
|
||||
endpoint: 'familybundle',
|
||||
@@ -284,6 +284,7 @@ const networks = [
|
||||
areaId: '3b4c609c-6a0d-4cb9-9cce-0605f32b79ec',
|
||||
blockId: 114458,
|
||||
scene: 'https://aziani.com',
|
||||
api: 'https://azianistudios.com/tour_api.php',
|
||||
cdn: 'https://c75c0c3063.mjedge.net',
|
||||
},
|
||||
},
|
||||
@@ -593,6 +594,11 @@ const networks = [
|
||||
concurrency: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'ksm',
|
||||
name: 'KSM',
|
||||
hasLogo: false,
|
||||
},
|
||||
{
|
||||
slug: 'letsdoeit',
|
||||
name: 'LetsDoeIt',
|
||||
@@ -777,6 +783,11 @@ const networks = [
|
||||
parentSession: false,
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'rsghmedia',
|
||||
name: 'RSGH Media',
|
||||
hasLogo: false,
|
||||
},
|
||||
{
|
||||
slug: 'score',
|
||||
name: 'SCORE',
|
||||
|
||||
+178
-12
@@ -6127,7 +6127,7 @@ const sites = [
|
||||
{
|
||||
slug: 'blackforwife',
|
||||
name: 'Black for Wife',
|
||||
url: 'https://www.blackforwife.com',
|
||||
url: 'https://www.blackforwife.org',
|
||||
parent: 'jayrock',
|
||||
parameters: {
|
||||
useGamma: true,
|
||||
@@ -6141,6 +6141,12 @@ const sites = [
|
||||
name: 'Jax Slayher',
|
||||
url: 'https://jaxslayher.com',
|
||||
},
|
||||
// JENNIFER WHITE
|
||||
{
|
||||
slug: 'jenniferwhite',
|
||||
name: 'Jennifer White',
|
||||
url: 'https://www.jenniferwhite.com',
|
||||
},
|
||||
// JESSE LOADS MONSTER FACIALS
|
||||
{
|
||||
slug: 'jesseloadsmonsterfacials',
|
||||
@@ -7379,6 +7385,16 @@ const sites = [
|
||||
studioId: 1214,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'Nick Morris Studio',
|
||||
slug: 'nickmorrisstudio',
|
||||
url: 'https://www.analvids.com/studios/nick_morris_studio',
|
||||
parent: 'analvids',
|
||||
hasLogo: false,
|
||||
parameters: {
|
||||
studioId: 1038,
|
||||
},
|
||||
},
|
||||
// LETS DOE IT
|
||||
{
|
||||
name: 'The White Boxxx',
|
||||
@@ -8501,6 +8517,33 @@ const sites = [
|
||||
id: 'hookup-pad',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'thepickupvan',
|
||||
name: 'The Pickup Van',
|
||||
url: 'https://www.mylf.com/series/the-pickup-van',
|
||||
parent: 'mylf',
|
||||
parameters: {
|
||||
id: 'the-pickup-van',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'motherfucker',
|
||||
name: 'Motherfucker',
|
||||
url: 'https://www.mylf.com/series/motherfucker',
|
||||
parent: 'mylf',
|
||||
parameters: {
|
||||
id: 'motherfucker',
|
||||
},
|
||||
},
|
||||
{
|
||||
slug: 'bluecollarbabes',
|
||||
name: 'Blue Collar Babes',
|
||||
url: 'https://www.mylf.com/series/blue-collar-babes',
|
||||
parent: 'mylf',
|
||||
parameters: {
|
||||
id: 'blue-collar-babes',
|
||||
},
|
||||
},
|
||||
/* premium */
|
||||
{
|
||||
slug: 'analmom',
|
||||
@@ -8535,12 +8578,9 @@ const sites = [
|
||||
{
|
||||
slug: 'mylfdom',
|
||||
name: 'MYLF Dom',
|
||||
url: 'https://www.mylfdom.com',
|
||||
url: 'https://www.mylf.com/series/mylfdom-ts',
|
||||
tags: ['bdsm', 'milf'],
|
||||
parent: 'mylf',
|
||||
parameters: {
|
||||
endpoint: 'mylf_dom',
|
||||
},
|
||||
},
|
||||
// NAUGHTY AMERICA
|
||||
{
|
||||
@@ -11098,6 +11138,59 @@ const sites = [
|
||||
slug: 'rickysroom',
|
||||
url: 'https://rickysroom.com',
|
||||
},
|
||||
// RSGH MEDIA
|
||||
{
|
||||
name: 'Family Therapy XXX',
|
||||
slug: 'familytherapyxxx',
|
||||
url: 'https://familytherapyxxx.com',
|
||||
tags: ['family'],
|
||||
parent: 'rsghmedia',
|
||||
independent: true,
|
||||
},
|
||||
{
|
||||
name: 'Mom Comes First',
|
||||
slug: 'momcomesfirst',
|
||||
url: 'https://momcomesfirst.com',
|
||||
tags: ['family'],
|
||||
parent: 'rsghmedia',
|
||||
independent: true,
|
||||
},
|
||||
{
|
||||
name: 'Perfect Girlfriend',
|
||||
slug: 'perfectgirlfriend',
|
||||
url: 'https://perfectgirlfriend.com',
|
||||
parent: 'rsghmedia',
|
||||
independent: true,
|
||||
},
|
||||
{
|
||||
name: 'Anal Therapy XXX',
|
||||
slug: 'analtherapyxxx',
|
||||
url: 'https://analtherapyxxx.com',
|
||||
tags: ['anal'],
|
||||
parent: 'rsghmedia',
|
||||
independent: true,
|
||||
},
|
||||
{
|
||||
name: 'Cock Ninja Studios',
|
||||
slug: 'cockninjastudios',
|
||||
url: 'https://cockninjastudios.com',
|
||||
parent: 'rsghmedia',
|
||||
independent: true,
|
||||
},
|
||||
{
|
||||
name: 'Wife Loves Black',
|
||||
slug: 'wifelovesblack',
|
||||
url: 'https://wifelovesblack.com',
|
||||
parent: 'rsghmedia',
|
||||
independent: true,
|
||||
},
|
||||
{
|
||||
name: 'Teen Loves Black',
|
||||
slug: 'teenlovesblack',
|
||||
url: 'https://teenlovesblack.com',
|
||||
parent: 'rsghmedia',
|
||||
independent: true,
|
||||
},
|
||||
// SCORE
|
||||
{
|
||||
name: '18 Eighteen',
|
||||
@@ -12328,13 +12421,6 @@ const sites = [
|
||||
url: 'https://www.teamskeet.com/series/bad-milfs',
|
||||
parent: 'teamskeet',
|
||||
},
|
||||
{
|
||||
slug: 'blackstepdad',
|
||||
name: 'Black StepDad',
|
||||
url: 'https://www.teamskeet.com/series/black-stepdad',
|
||||
alias: ['bsd'],
|
||||
parent: 'teamskeet',
|
||||
},
|
||||
{
|
||||
slug: 'exxxtrasmall',
|
||||
name: 'Exxxtra Small',
|
||||
@@ -12714,6 +12800,44 @@ const sites = [
|
||||
url: 'https://www.teamskeet.com/series/passport-bros',
|
||||
parent: 'teamskeet',
|
||||
},
|
||||
{
|
||||
slug: 'underthetable',
|
||||
name: 'Under the Table',
|
||||
url: 'https://www.teamskeet.com/series/under-the-table',
|
||||
parent: 'teamskeet',
|
||||
},
|
||||
{
|
||||
slug: 'mydirtyuncle',
|
||||
name: 'My Dirty Uncle',
|
||||
url: 'https://www.teamskeet.com/series/my-dirty-uncle',
|
||||
tags: ['family'],
|
||||
parent: 'teamskeet',
|
||||
},
|
||||
{
|
||||
slug: 'mormongirlz',
|
||||
name: 'Mormon Girlz',
|
||||
url: 'https://www.teamskeet.com/series/mormon-girlz',
|
||||
tags: ['religion'],
|
||||
parent: 'teamskeet',
|
||||
},
|
||||
{
|
||||
slug: 'breedingmaterial',
|
||||
name: 'Breeding Material',
|
||||
url: 'https://www.teamskeet.com/series/breeding-material',
|
||||
parent: 'teamskeet',
|
||||
},
|
||||
{
|
||||
slug: 'teamskeetsingles',
|
||||
name: 'TeamSkeet Singles',
|
||||
url: 'https://www.teamskeet.com/series/teamskeet-singles',
|
||||
parent: 'teamskeet',
|
||||
},
|
||||
{
|
||||
slug: 'sheseducedme',
|
||||
name: 'She Seduced Me',
|
||||
url: 'https://www.teamskeet.com/series/she-seduced-me',
|
||||
parent: 'teamskeet',
|
||||
},
|
||||
// PERVZ
|
||||
{
|
||||
slug: 'pervzfeatures',
|
||||
@@ -12790,6 +12914,12 @@ const sites = [
|
||||
url: 'https://www.pervz.com/series/pervdriver',
|
||||
parent: 'pervz',
|
||||
},
|
||||
{
|
||||
slug: 'charmed',
|
||||
name: 'Charmed',
|
||||
url: 'https://www.pervz.com/series/charmed',
|
||||
parent: 'pervz',
|
||||
},
|
||||
// SWAPPZ
|
||||
{
|
||||
slug: 'swappzfeatures',
|
||||
@@ -12856,6 +12986,13 @@ const sites = [
|
||||
parent: 'freeuse',
|
||||
tags: ['free-use', 'pov'],
|
||||
},
|
||||
{
|
||||
slug: 'freeusesingles',
|
||||
name: 'Freeuse Singles',
|
||||
url: 'https://www.freeuse.com/series/freeuse-singles',
|
||||
parent: 'freeuse',
|
||||
tags: ['free-use'],
|
||||
},
|
||||
// FAMILY STROKES
|
||||
{
|
||||
slug: 'familystrokes',
|
||||
@@ -12937,6 +13074,14 @@ const sites = [
|
||||
tags: ['family', 'milf'],
|
||||
parent: 'familystrokes',
|
||||
},
|
||||
{
|
||||
slug: 'blackstepdad',
|
||||
name: 'Black StepDad',
|
||||
url: 'https://www.familystrokes.com/series/black-stepdad',
|
||||
alias: ['bsd'],
|
||||
tags: ['family'],
|
||||
parent: 'familystrokes',
|
||||
},
|
||||
// SAY UNCLE
|
||||
{
|
||||
slug: 'blackgodz',
|
||||
@@ -13529,6 +13674,27 @@ const sites = [
|
||||
},
|
||||
},
|
||||
},
|
||||
// SKM
|
||||
{
|
||||
slug: 'cosplayground',
|
||||
name: 'Cosplayground',
|
||||
url: 'https://cosplayground.com',
|
||||
tags: ['cosplay'],
|
||||
parent: 'ksm',
|
||||
independent: true,
|
||||
parameters: {
|
||||
// seems to use same CMS as Aziani
|
||||
api: 'https://api.cosplayground.com/tour_api.php',
|
||||
seriesId: null,
|
||||
areaId: '2bc2444e-a053-40a2-bc4b-ca3618521563',
|
||||
cdn: 'https://c880f36aa8.mjedge.net',
|
||||
entryIdSlug: true,
|
||||
videos: '/videos',
|
||||
scenePath: '/videos',
|
||||
modelPath: '/models',
|
||||
imageType: 'photos',
|
||||
},
|
||||
},
|
||||
// TEEN CORE CLUB
|
||||
{
|
||||
name: 'Analyzed Girls',
|
||||
|
||||
@@ -706,6 +706,7 @@ const tagMedia = [
|
||||
['blowjob', 'azul_hermosa_realitykings', 'Azul Hermosa and Scott Nails in "Diva For A Day"', 'brazzers'],
|
||||
['blowjob', 3, 'Rose Valie', 'handsonhardcore'],
|
||||
['blowjob', 2, 'Luna Kitsuen in "Gag Reflex"', 'evilangel'],
|
||||
['body-writing', 'emma_rosie_julesjordan', 'Emma Rosie', 'julesjordan'],
|
||||
['bondage', 0, 'Veronica Leal', 'herlimit'],
|
||||
['brunette', 0, 'Darcie Dolce', 'playboy'],
|
||||
['bts', 'charlie_red_private', 'Charlie Red', 'private'],
|
||||
|
||||
+9
-2
@@ -611,8 +611,15 @@ async function upsertProfiles(profiles) {
|
||||
media_id: profile.avatarMediaId,
|
||||
}));
|
||||
|
||||
if (avatars.length > 0) {
|
||||
await batchInsert('actors_avatars', avatars, { conflict: false });
|
||||
const resolvedAvatars = avatars.filter((avatar) => !!avatar.profile_id);
|
||||
const unresolvedAvatars = avatars.filter((avatar) => !avatar.profile_id);
|
||||
|
||||
unresolvedAvatars.forEach((avatar) => {
|
||||
logger.warn(`Skipping avatar for actor ${avatar.actor_id} (${avatar.media_id}), no profile could be resolved`);
|
||||
});
|
||||
|
||||
if (resolvedAvatars.length > 0) {
|
||||
await batchInsert('actors_avatars', resolvedAvatars, { conflict: false });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+32
-40
@@ -9,7 +9,6 @@ const argv = require('./argv');
|
||||
const include = require('./utils/argv-include')(argv);
|
||||
const { fetchReleaseEntities, urlToHostname } = require('./entities');
|
||||
const logger = require('./logger')(__filename);
|
||||
const qu = require('./utils/qu');
|
||||
const getRecursiveParameters = require('./utils/get-recursive-parameters');
|
||||
const windows = require('./utils/http-windows');
|
||||
|
||||
@@ -85,35 +84,7 @@ async function fetchScene(scraper, url, entity, baseRelease, options, type = 'sc
|
||||
}
|
||||
|
||||
if ((type === 'scene' && scraper.scrapeScene) || (type === 'movie' && scraper.scrapeMovie)) {
|
||||
/*
|
||||
if (scraper.useUnprint || (type === 'scene' && scraper.scrapeScene?.unprint) || (type === 'movie' && scraper.scrapeMovie?.unprint)) {
|
||||
return fetchUnprintScene(scraper, url, entity, baseRelease, options, type);
|
||||
}
|
||||
*/
|
||||
|
||||
if (!scraper.deprecated) {
|
||||
return fetchUnprintScene(scraper, url, entity, baseRelease, options, type);
|
||||
}
|
||||
|
||||
const session = qu.session();
|
||||
|
||||
const res = await qu.get(url, null, null, {
|
||||
session,
|
||||
rejectUnauthorized: false,
|
||||
});
|
||||
|
||||
const cookie = await session._sessionOptions.cookieJar.get(url);
|
||||
|
||||
if (res.ok) {
|
||||
return scraper[type === 'movie' ? 'scrapeMovie' : 'scrapeScene'](res.item, url, entity, baseRelease, options, {
|
||||
session,
|
||||
headers: res.headers,
|
||||
cookieJar: session._sessionOptions.cookieJar,
|
||||
cookie,
|
||||
});
|
||||
}
|
||||
|
||||
return res.status;
|
||||
return fetchUnprintScene(scraper, url, entity, baseRelease, options, type);
|
||||
}
|
||||
|
||||
return null;
|
||||
@@ -123,15 +94,21 @@ function fetchMovie(scraper, url, entity, baseRelease, options) {
|
||||
return fetchScene(scraper, url, entity, baseRelease, options, 'movie');
|
||||
}
|
||||
|
||||
async function scrapeRelease(baseRelease, entitiesByHostname, type = 'scene') {
|
||||
const entity = baseRelease.entity || entitiesByHostname[urlToHostname(baseRelease.url)];
|
||||
function shouldDeepScrape(baseRelease) {
|
||||
return argv.deep && !!(baseRelease.url || baseRelease.path || baseRelease.forceDeep);
|
||||
}
|
||||
|
||||
async function scrapeRelease(baseRelease, entities, type = 'scene') {
|
||||
const entity = entities.byId[baseRelease.entity?.id]
|
||||
|| entities.byHostname[urlToHostname(baseRelease.url)]
|
||||
|| baseRelease.entity;
|
||||
|
||||
if (!entity) {
|
||||
logger.warn(`No entity available for ${baseRelease.url}`);
|
||||
return baseRelease;
|
||||
}
|
||||
|
||||
if ((!baseRelease.url && !baseRelease.path && !baseRelease.forceDeep) || !argv.deep) {
|
||||
if (!shouldDeepScrape(baseRelease)) {
|
||||
return {
|
||||
...baseRelease,
|
||||
entity,
|
||||
@@ -247,22 +224,37 @@ async function scrapeRelease(baseRelease, entitiesByHostname, type = 'scene') {
|
||||
}
|
||||
|
||||
async function scrapeReleases(baseReleases, entitiesByHostname, type) {
|
||||
const entitiesWithBeforeDataEntries = await Promise.all(Object.entries(entitiesByHostname).map(async ([slug, entity]) => {
|
||||
if (entity.scraper?.beforeFetchScenes) {
|
||||
// entities attached to base releases (e.g. from updates) are not included in entitiesByHostname
|
||||
const entitiesById = Object.fromEntries([
|
||||
...Object.values(entitiesByHostname),
|
||||
...baseReleases.map((baseRelease) => baseRelease.entity),
|
||||
].filter(Boolean).map((entity) => [entity.id, entity]));
|
||||
|
||||
// only run before hook for entities that will actually be deep scraped
|
||||
const deepEntityIds = new Set(baseReleases
|
||||
.filter((baseRelease) => shouldDeepScrape(baseRelease))
|
||||
.map((baseRelease) => baseRelease.entity?.id || entitiesByHostname[urlToHostname(baseRelease.url)]?.id)
|
||||
.filter(Boolean));
|
||||
|
||||
const entitiesWithBeforeDataById = Object.fromEntries(await Promise.all(Object.values(entitiesById).map(async (entity) => {
|
||||
if (deepEntityIds.has(entity.id) && entity.scraper?.beforeFetchScenes) {
|
||||
const parameters = getRecursiveParameters(entity);
|
||||
const preData = await entity.scraper.beforeFetchScenes(entity, parameters);
|
||||
|
||||
return [slug, { ...entity, preData }];
|
||||
return [entity.id, { ...entity, preData }];
|
||||
}
|
||||
|
||||
return [slug, entity];
|
||||
}));
|
||||
return [entity.id, entity];
|
||||
})));
|
||||
|
||||
const entitiesWithBeforeDataBySlug = Object.fromEntries(entitiesWithBeforeDataEntries.filter(Boolean));
|
||||
const entitiesWithBeforeData = {
|
||||
byId: entitiesWithBeforeDataById,
|
||||
byHostname: Object.fromEntries(Object.entries(entitiesByHostname).map(([hostname, entity]) => [hostname, entitiesWithBeforeDataById[entity.id]])),
|
||||
};
|
||||
|
||||
return Promise.map(
|
||||
baseReleases,
|
||||
async (baseRelease) => scrapeRelease(baseRelease, entitiesWithBeforeDataBySlug, type),
|
||||
async (baseRelease) => scrapeRelease(baseRelease, entitiesWithBeforeData, type),
|
||||
{ concurrency: 1 },
|
||||
);
|
||||
}
|
||||
|
||||
+3
-3
@@ -15,6 +15,7 @@ const sharp = require('sharp');
|
||||
const blake2 = require('blake2');
|
||||
const { format } = require('date-fns');
|
||||
const taskQueue = require('promise-task-queue');
|
||||
const unprint = require('unprint');
|
||||
|
||||
const { Upload } = require('@aws-sdk/lib-storage');
|
||||
const { S3Client, DeleteObjectsCommand } = require('@aws-sdk/client-s3');
|
||||
@@ -25,7 +26,6 @@ const knex = require('./knex');
|
||||
const http = require('./utils/http');
|
||||
const batchInsert = require('./utils/batch-insert');
|
||||
const chunk = require('./utils/chunk');
|
||||
const { get } = require('./utils/qu');
|
||||
const { fetchEntityReleaseIds } = require('./entity-releases');
|
||||
|
||||
const streamQueue = taskQueue();
|
||||
@@ -383,10 +383,10 @@ async function extractSource(baseSource, { existingExtractMediaByUrl }) {
|
||||
};
|
||||
}
|
||||
|
||||
const res = await get(baseSource.url);
|
||||
const res = await unprint.get(baseSource.url);
|
||||
|
||||
if (res.ok) {
|
||||
const src = await baseSource.extract(res.item);
|
||||
const src = await baseSource.extract(res.context);
|
||||
|
||||
return {
|
||||
...baseSource,
|
||||
|
||||
@@ -29,7 +29,6 @@ const karups = require('./karups');
|
||||
const kellymadison = require('./kellymadison');
|
||||
const kink = require('./kink');
|
||||
const mariskax = require('./mariskax');
|
||||
// const analvids = require('./analvids');
|
||||
const pornbox = require('./pornbox');
|
||||
const littlecapricedreams = require('./littlecapricedreams');
|
||||
const loveherfilms = require('./loveherfilms');
|
||||
@@ -59,7 +58,6 @@ const teenmegaworld = require('./teenmegaworld');
|
||||
const testedefudelidade = require('./testedefudelidade');
|
||||
const theflourish = require('./theflourish');
|
||||
const tokyohot = require('./tokyohot');
|
||||
// const topwebmodels = require('./topwebmodels');
|
||||
const virtualtaboo = require('./virtualtaboo');
|
||||
const vixen = require('./vixen');
|
||||
const wankzvr = require('./wankzvr');
|
||||
@@ -229,7 +227,6 @@ module.exports = {
|
||||
boyfun: karups,
|
||||
kellymadison,
|
||||
'8kmembers': kellymadison,
|
||||
// analvids,
|
||||
analvids: pornbox,
|
||||
littlecapricedreams,
|
||||
mariskax,
|
||||
@@ -279,6 +276,7 @@ module.exports = {
|
||||
// etc
|
||||
'18vr': badoink,
|
||||
theflourishxxx: theflourish,
|
||||
cosplayground: aziani,
|
||||
pierrewoodman,
|
||||
exploitedx, // only from known URL that will specify site
|
||||
fullpornnetwork,
|
||||
|
||||
@@ -62,7 +62,7 @@ async function scrapeRelease({ query, html, element }, { url, entity, baseReleas
|
||||
};
|
||||
});
|
||||
|
||||
release.tags = query.contents('.tags a, .categories a');
|
||||
release.tags = query.contents('.tags a, .categories a, .tags + div a[data-label="Attribute"]');
|
||||
release.studio = parameters?.studio === false ? null : slugify(query.content('.studio span:last-child, .studio a'), '');
|
||||
|
||||
if (type === 'scene') {
|
||||
@@ -103,6 +103,18 @@ async function scrapeRelease({ query, html, element }, { url, entity, baseReleas
|
||||
release.qualities = [2160];
|
||||
}
|
||||
|
||||
const serieTitle = query.content('.series a');
|
||||
const serieUrl = query.url('.series a');
|
||||
const serieId = serieUrl && new URL(serieUrl).searchParams.get('series');
|
||||
|
||||
if (serieTitle && serieId) {
|
||||
release.serie = {
|
||||
entryId: serieId,
|
||||
title: serieTitle,
|
||||
url: serieUrl,
|
||||
};
|
||||
}
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,45 +1,45 @@
|
||||
'use strict';
|
||||
|
||||
const { fetchLatest, fetchProfile } = require('./julesjordan');
|
||||
const unprint = require('unprint');
|
||||
|
||||
const qu = require('../utils/q');
|
||||
const { fetchLatest, fetchProfile } = require('./julesjordan');
|
||||
const slugify = require('../utils/slugify');
|
||||
|
||||
function scrapeScene({ query }, channel) {
|
||||
const release = {};
|
||||
const title = query.cnt('.update_title', false);
|
||||
const title = query.content('.update_title', { trim: false });
|
||||
|
||||
release.title = title.trim();
|
||||
release.description = query.cnt('.latest_update_description');
|
||||
release.description = query.content('.latest_update_description');
|
||||
|
||||
release.date = query.date('.update_date', 'MM/DD/YYYY');
|
||||
release.actors = query.all('.tour_update_models a').map((actorEl) => ({
|
||||
name: query.cnt(actorEl),
|
||||
url: query.url(actorEl, null),
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null),
|
||||
}));
|
||||
|
||||
release.entryId = slugify(release.title);
|
||||
|
||||
release.duration = Number(query.cnt('.update_counts_preview_table').match(/(\d+) min/)[1]) * 60;
|
||||
release.duration = Number(query.content('.update_counts_preview_table').match(/(\d+) min/)[1]) * 60;
|
||||
|
||||
const poster = query.img('.large_update_thumb', 'src', { origin: channel.url });
|
||||
const trailer = query.q('.update_image a', 'onclick')?.match(/'(.+)'/)?.[1];
|
||||
const poster = query.img('.large_update_thumb', { attribute: 'src', origin: channel.url });
|
||||
const trailer = query.attribute('.update_image a', 'onclick')?.match(/'(.+)'/)?.[1];
|
||||
|
||||
release.poster = [
|
||||
poster.replace(/.jpg$/, '-full.jpg'),
|
||||
poster,
|
||||
];
|
||||
|
||||
release.photos = query.imgs('.small_update_thumb', 'src', { origin: channel.url }).map((img) => [
|
||||
release.photos = query.imgs('.small_update_thumb', { attribute: 'src', origin: channel.url }).map((img) => [
|
||||
img.replace(/.jpg$/, '-full.jpg'),
|
||||
img,
|
||||
]);
|
||||
|
||||
if (trailer) {
|
||||
release.trailer = qu.prefixUrl(encodeURI(trailer), channel.url);
|
||||
release.trailer = unprint.prefixUrl(encodeURI(trailer), channel.url);
|
||||
}
|
||||
|
||||
release.tags = query.cnts('.tour_update_tags a');
|
||||
release.tags = query.contents('.tour_update_tags a');
|
||||
|
||||
return release;
|
||||
}
|
||||
@@ -49,10 +49,10 @@ async function fetchLatestLocal(channel, page, include, preData) {
|
||||
}
|
||||
|
||||
async function fetchScene(url, channel) {
|
||||
const res = await qu.get(url);
|
||||
const res = await unprint.get(url);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeScene(res.item, channel);
|
||||
return scrapeScene(res.context, channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
|
||||
@@ -1,174 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const unprint = require('unprint');
|
||||
|
||||
const slugify = require('../utils/slugify');
|
||||
|
||||
function extractTitle(originalTitle) {
|
||||
const titleComponents = originalTitle.split(' ');
|
||||
// const sceneIdMatch = titleComponents.slice(-1)[0].match(/(AB|AF|GP|SZ|IV|GIO|RS|TW|MA|FM|SAL|NR|AA|GL|BZ|FS|KS|OTS|NF|NT|AX|RV|CM|BTG)\d+/); // detect studio prefixes
|
||||
const sceneIdMatch = titleComponents.slice(-1)[0].match(/\w+\d+\s*$/); // detect studio prefixes
|
||||
const shootId = sceneIdMatch ? sceneIdMatch[0] : null;
|
||||
const title = sceneIdMatch ? titleComponents.slice(0, -1).join(' ') : originalTitle;
|
||||
|
||||
return { shootId, title };
|
||||
}
|
||||
|
||||
function scrapeAll(scenes, channel) {
|
||||
return scenes.map(({ query }) => {
|
||||
const release = {};
|
||||
|
||||
release.url = query.url('.card-scene__view > a');
|
||||
release.entryId = query.dataset(null, 'content') || new URL(release.url).pathname.match(/watch\/(\d+)/)?.[1];
|
||||
|
||||
release.title = query.content('.card-scene__text');
|
||||
release.shootId = extractTitle(release.title).shootId;
|
||||
|
||||
release.date = query.date('.label--time:nth-child(2)', 'YYYY-MM-DD'); // only available on front-page, not on studio page
|
||||
release.duration = query.duration('.label--time:first-child');
|
||||
|
||||
release.poster = query.img('.card-scene__view img', { attribute: 'data-src' });
|
||||
const caps = query.json('.card-scene__view > a', { attribute: 'data-casting' })?.map((timestamp) => `${channel.url}/casting/${release.entryId}/${timestamp}`);
|
||||
|
||||
if (caps && release.poster) {
|
||||
release.caps = caps;
|
||||
} else if (caps) {
|
||||
[release.poster, ...release.caps] = caps;
|
||||
}
|
||||
|
||||
release.teaser = query.video('.card-scene__view > a', { attribute: 'data-preview' });
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page) {
|
||||
// const res = await unprint.get(`https://www.analvids.com/new-videos/${page}`, { selectAll: '.card-scene' }); // analvids as channel
|
||||
// studios as channels
|
||||
const url = `${channel.url}/latest/${page}`;
|
||||
|
||||
const res = await unprint.get(url, {
|
||||
selectAll: '.card-scene',
|
||||
headers: {
|
||||
Referer: url,
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeAll(res.context, channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
function scrapeScene({ query }, url) {
|
||||
const release = {};
|
||||
|
||||
release.entryId = new URL(url).pathname.match(/watch\/(\d+)/)?.[1];
|
||||
|
||||
const featuring = query.content('.watch__title .watch__featuring_models');
|
||||
|
||||
release.title = query.content('.watch__title').replace(featuring, '');
|
||||
release.description = query.content('.text-mob-more');
|
||||
release.shootId = extractTitle(release.title).shootId;
|
||||
|
||||
release.date = query.date('.bi-calendar3', 'YYYY-MM-DD');
|
||||
release.duration = query.duration('.bi-clock');
|
||||
|
||||
release.actors = query.all('.watch__title a[href*="/model"]').map((el) => ({
|
||||
name: unprint.query.content(el),
|
||||
url: unprint.query.url(el, null),
|
||||
}));
|
||||
|
||||
release.tags = query.contents('.genres-list a[href*="/genre"]');
|
||||
|
||||
// release.studio = slugify(query.content('.genres-list a[href*="/studios"]'), '', { removePunctuation: true });
|
||||
|
||||
release.poster = query.img('.watch__video video', { attribute: 'data-poster' });
|
||||
|
||||
release.trailer = query.all('.watch__video source').map((el) => ({
|
||||
src: unprint.query.video(el, null, { attribute: 'src' }),
|
||||
quality: unprint.query.number(el, null, { attribute: 'size' }),
|
||||
}));
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
async function fetchScene(url) {
|
||||
const res = await unprint.get(url, {
|
||||
headers: {
|
||||
Referer: url,
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeScene(res.context, url);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
function scrapeProfile({ query }, url, channel) {
|
||||
const profile = { url };
|
||||
|
||||
profile.nationality = query.content('.model__info a[href*="/nationality"]');
|
||||
profile.age = query.number('//td[contains(text(), "Age")]/following-sibling::td');
|
||||
|
||||
profile.avatar = query.img('.model__left img');
|
||||
|
||||
profile.scenes = scrapeAll(unprint.initAll(query.all('.card-scene')), channel);
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function getActorUrl(actor, channel) {
|
||||
if (actor.url) {
|
||||
return actor.url;
|
||||
}
|
||||
|
||||
const searchUrl = `${channel.url}/api/autocomplete/search?q=${slugify(actor.name, '+')}`;
|
||||
|
||||
const searchRes = await unprint.get(searchUrl, {
|
||||
headers: {
|
||||
Referer: actor.url,
|
||||
},
|
||||
});
|
||||
|
||||
if (!searchRes.ok) {
|
||||
return searchRes.status;
|
||||
}
|
||||
|
||||
const result = searchRes.body.terms.find((item) => item.type === 'model');
|
||||
|
||||
if (result) {
|
||||
return result.url;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function fetchProfile(actor, { channel }) {
|
||||
const actorUrl = await getActorUrl(actor, channel);
|
||||
|
||||
if (typeof actorUrl !== 'string') {
|
||||
return actorUrl;
|
||||
}
|
||||
|
||||
const bioRes = await unprint.get(actorUrl, {
|
||||
headers: {
|
||||
Referer: actor.url,
|
||||
},
|
||||
});
|
||||
|
||||
if (bioRes.ok) {
|
||||
return scrapeProfile(bioRes.context, actorUrl, channel);
|
||||
}
|
||||
|
||||
return bioRes.status;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchScene,
|
||||
fetchProfile,
|
||||
};
|
||||
@@ -1,203 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const { JSDOM } = require('jsdom');
|
||||
const cheerio = require('cheerio');
|
||||
const moment = require('moment');
|
||||
|
||||
const http = require('../utils/http');
|
||||
const slugify = require('../utils/slugify');
|
||||
|
||||
function extractTitle(originalTitle) {
|
||||
const titleComponents = originalTitle.split(' ');
|
||||
// const sceneIdMatch = titleComponents.slice(-1)[0].match(/(AB|AF|GP|SZ|IV|GIO|RS|TW|MA|FM|SAL|NR|AA|GL|BZ|FS|KS|OTS|NF|NT|AX|RV|CM|BTG)\d+/); // detect studio prefixes
|
||||
const sceneIdMatch = titleComponents.slice(-1)[0].match(/\w+\d+\s*$/); // detect studio prefixes
|
||||
const shootId = sceneIdMatch ? sceneIdMatch[0] : null;
|
||||
const title = sceneIdMatch ? titleComponents.slice(0, -1).join(' ') : originalTitle;
|
||||
|
||||
return { shootId, title };
|
||||
}
|
||||
|
||||
function getPoster(posterElement, sceneId) {
|
||||
const posterStyle = posterElement.attr('style');
|
||||
|
||||
if (posterStyle) {
|
||||
return posterStyle.slice(posterStyle.indexOf('(') + 1, -1);
|
||||
}
|
||||
|
||||
const posterRange = posterElement.attr('data-casting');
|
||||
const posterRangeData = posterRange ? JSON.parse(posterRange) : null;
|
||||
const posterTimeRange = posterRangeData[Math.floor(Math.random() * posterRangeData.length)];
|
||||
|
||||
if (!posterTimeRange) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (typeof posterTimeRange === 'number') {
|
||||
// poster time is already a single time value
|
||||
return `https://analvids.com/casting/${sceneId}/${posterTimeRange}`;
|
||||
}
|
||||
|
||||
const [max, min] = posterTimeRange.split('-');
|
||||
const posterTime = Math.floor(Math.random() * (Number(max) - Number(min) + 1) + Number(min));
|
||||
|
||||
return `https://analvids.com/casting/${sceneId}/${posterTime}`;
|
||||
}
|
||||
|
||||
function scrapeAll(html) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
const scenesElements = $('.thumbnails > div').toArray();
|
||||
|
||||
return scenesElements.map((element) => {
|
||||
const sceneLinkElement = $(element).find('.thumbnail-title a');
|
||||
const url = sceneLinkElement.attr('href');
|
||||
|
||||
const originalTitle = sceneLinkElement.text().trim(); // title attribute breaks when they use \\ escaping
|
||||
const { shootId, title } = extractTitle(originalTitle);
|
||||
const entryId = new URL(url).pathname.split('/')[2];
|
||||
|
||||
const date = moment.utc($(element).attr('release'), 'YYYY/MM/DD').toDate();
|
||||
|
||||
const sceneId = $(element).attr('data-content');
|
||||
const posterElement = $(element).find('.thumbnail-avatar');
|
||||
|
||||
const poster = getPoster(posterElement, sceneId);
|
||||
|
||||
return {
|
||||
url,
|
||||
shootId,
|
||||
entryId,
|
||||
title,
|
||||
date,
|
||||
poster,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function scrapeScene(html, url, site, useGallery) {
|
||||
const $ = cheerio.load(html, { normalizeWhitespace: true });
|
||||
const playerObject = $('script:contains("new WatchPage")').html();
|
||||
const playerData = playerObject && playerObject.slice(playerObject.indexOf('{"swf":'), playerObject.lastIndexOf('},') + 1);
|
||||
const data = playerData && JSON.parse(playerData);
|
||||
|
||||
const release = { url };
|
||||
|
||||
const originalTitle = $('h1.watchpage-title').text().trim();
|
||||
const { shootId, title } = extractTitle(originalTitle);
|
||||
|
||||
release.shootId = shootId;
|
||||
release.entryId = new URL(url).pathname.split('/')[2];
|
||||
|
||||
release.title = title;
|
||||
release.date = moment.utc($('span[title="Release date"] a').text(), 'YYYY-MM-DD').toDate();
|
||||
|
||||
const [actorsElement, tagsElement, descriptionElement] = $('.scene-description__row').toArray();
|
||||
|
||||
release.description = $('meta[name="description"]')?.attr('content')?.trim()
|
||||
|| (descriptionElement && $(descriptionElement).find('dd').text().trim());
|
||||
|
||||
release.actors = $(actorsElement)
|
||||
.find('a[href*="com/model"]')
|
||||
.map((actorIndex, actorElement) => $(actorElement).text()).toArray();
|
||||
|
||||
release.duration = moment.duration($('span[title="Runtime"]').text().trim()).asSeconds();
|
||||
release.tags = $(tagsElement).find('a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
|
||||
|
||||
const photos = useGallery
|
||||
? $('.gallery a img').map((photoIndex, photoElement) => $(photoElement).attr('src')).toArray()
|
||||
: $('.screenshots img').map((photoIndex, photoElement) => $(photoElement).attr('src')).toArray();
|
||||
|
||||
release.photos = photos.map((source) => {
|
||||
// source without parameters sometimes serves larger preview photo
|
||||
const { origin, pathname } = new URL(source);
|
||||
|
||||
return `${origin}${pathname}`;
|
||||
});
|
||||
|
||||
const posterStyle = $('#player').attr('style');
|
||||
const poster = posterStyle.slice(posterStyle.indexOf('(') + 1, -1);
|
||||
|
||||
release.poster = poster || release.photos.slice(Math.floor(release.photos.length / 3) * -1); // poster unavailable, try last 1/3rd of high res photos as fallback
|
||||
|
||||
if (data) {
|
||||
const qualityMap = {
|
||||
web: 240,
|
||||
vga: 480,
|
||||
hd: 720,
|
||||
'1080p': 1080,
|
||||
};
|
||||
|
||||
release.trailer = data.clip.qualities.map((trailer) => ({
|
||||
src: trailer.src,
|
||||
type: trailer.type,
|
||||
quality: qualityMap[trailer.quality] || trailer.quality,
|
||||
}));
|
||||
}
|
||||
|
||||
const studioName = $('.watchpage-studioname').first().text().trim();
|
||||
release.studio = slugify(studioName, '', { removePunctuation: true });
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
async function scrapeProfile(html, _url, actorName) {
|
||||
const { document } = new JSDOM(html).window;
|
||||
|
||||
const profile = {
|
||||
name: actorName,
|
||||
};
|
||||
|
||||
const avatarEl = document.querySelector('.model--avatar img[src^="http"]');
|
||||
const entries = Array.from(document.querySelectorAll('.model--description tr'), (el) => el.textContent.replace(/\n/g, '').split(':'));
|
||||
|
||||
const bio = entries
|
||||
.filter((entry) => entry.length === 2) // ignore entries without ':' (About section, see Blanche Bradburry)
|
||||
.reduce((acc, [key, value]) => ({ ...acc, [key.trim()]: value.trim() }), {});
|
||||
|
||||
profile.birthPlace = bio.Nationality;
|
||||
|
||||
if (bio.Age) profile.age = bio.Age;
|
||||
if (avatarEl) profile.avatar = avatarEl.src;
|
||||
|
||||
profile.releases = scrapeAll(html);
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const res = await http.get(`${site.url}/new-videos/${page}`);
|
||||
|
||||
return scrapeAll(res.body.toString(), site);
|
||||
}
|
||||
|
||||
async function fetchScene(url, site) {
|
||||
const useGallery = true;
|
||||
|
||||
// TODO: fall back on screenshots when gallery is not available
|
||||
const res = useGallery
|
||||
? await http.get(`${url}/gallery#gallery`)
|
||||
: await http.get(`${url}/screenshots#screenshots`);
|
||||
|
||||
return scrapeScene(res.body.toString(), url, site, useGallery);
|
||||
}
|
||||
|
||||
async function fetchProfile({ name: actorName }) {
|
||||
const res = await http.get(`https://www.analvids.com/api/autocomplete/search?q=${actorName.replace(' ', '+')}`);
|
||||
const data = res.body;
|
||||
|
||||
const result = data.terms.find((item) => item.type === 'model');
|
||||
|
||||
if (result) {
|
||||
const bioRes = await http.get(result.url);
|
||||
const html = bioRes.body.toString();
|
||||
|
||||
return scrapeProfile(html, result.url, actorName);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchProfile,
|
||||
fetchScene,
|
||||
};
|
||||
+30
-27
@@ -1,6 +1,9 @@
|
||||
'use strict';
|
||||
|
||||
const { get, geta, ctxa } = require('../utils/q');
|
||||
const unprint = require('unprint');
|
||||
|
||||
// paths are relative, read raw attributes instead of resolved URLs
|
||||
const raw = { forceGetAttribute: true };
|
||||
|
||||
function extractActors(actorString) {
|
||||
return actorString
|
||||
@@ -20,48 +23,48 @@ function matchActors(actorString, models) {
|
||||
}
|
||||
|
||||
function scrapeLatest(scenes, site, models) {
|
||||
return scenes.map(({ qu }) => {
|
||||
return scenes.map(({ query }) => {
|
||||
const release = {};
|
||||
|
||||
const pathname = qu.url('a.itemimg').slice(1);
|
||||
const pathname = query.attribute('a.itemimg', 'href', raw).slice(1);
|
||||
[release.entryId] = pathname.split('/').slice(-1);
|
||||
release.url = `${site.url}${pathname}`;
|
||||
|
||||
release.title = qu.q('.itemimg img', 'alt') || qu.q('h4 a', true);
|
||||
release.description = qu.q('.mas_longdescription', true);
|
||||
release.date = qu.date('.movie_info2', 'MM/DD/YY', /\d{2}\/\d{2}\/\d{2}/);
|
||||
release.title = query.attribute('.itemimg img', 'alt') || query.content('h4 a');
|
||||
release.description = query.content('.mas_longdescription');
|
||||
release.date = query.date('.movie_info2', 'MM/DD/YY', { match: /\d{2}\/\d{2}\/\d{2}/ });
|
||||
|
||||
const actorString = qu.q('.mas_description', true);
|
||||
const actorString = query.content('.mas_description');
|
||||
const actors = matchActors(actorString, models);
|
||||
|
||||
if (actors.length > 0) release.actors = actors;
|
||||
else release.actors = extractActors(actorString);
|
||||
|
||||
const posterPath = qu.img('.itemimg img');
|
||||
const posterPath = query.attribute('.itemimg img', 'src', raw);
|
||||
release.poster = `${site.url}/${posterPath}`;
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
function scrapeScene({ html, qu }, url, site, include, models) {
|
||||
function scrapeScene({ html, query }, url, site, include, models) {
|
||||
const release = { url };
|
||||
|
||||
[release.entryId] = url.split('/').slice(-1);
|
||||
release.title = qu.q('.mas_title', true);
|
||||
release.description = qu.q('.mas_longdescription', true);
|
||||
release.date = qu.date('.mas_description', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
|
||||
release.title = query.content('.mas_title');
|
||||
release.description = query.content('.mas_longdescription');
|
||||
release.date = query.date('.mas_description', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
|
||||
|
||||
if (models) {
|
||||
const actorString = qu.q('.mas_description', true).replace(/\w+ \d{1,2}, \d{4}/, '');
|
||||
const actorString = query.content('.mas_description').replace(/\w+ \d{1,2}, \d{4}/, '');
|
||||
const actors = matchActors(actorString, models);
|
||||
if (actors.length > 0) release.actors = actors;
|
||||
else release.actors = extractActors(actorString);
|
||||
}
|
||||
|
||||
release.tags = qu.all('.tags a', true);
|
||||
release.tags = query.contents('.tags a');
|
||||
|
||||
release.photos = qu.imgs('.stills img').map((photoPath) => `${site.url}/${photoPath}`);
|
||||
release.photos = query.attributes('.stills img', 'src', raw).map((photoPath) => `${site.url}/${photoPath}`);
|
||||
|
||||
const posterIndex = 'splash:';
|
||||
const poster = html.slice(html.indexOf('faceimages/', posterIndex), html.indexOf('.jpg', posterIndex) + 4);
|
||||
@@ -74,20 +77,20 @@ function scrapeScene({ html, qu }, url, site, include, models) {
|
||||
return release;
|
||||
}
|
||||
|
||||
function extractModels({ el }, site) {
|
||||
const models = ctxa(el, '.item');
|
||||
function extractModels({ element }, site) {
|
||||
const models = unprint.initAll(element, '.item');
|
||||
|
||||
return models.map(({ qu }) => {
|
||||
return models.map(({ query }) => {
|
||||
const actor = { gender: 'female' };
|
||||
|
||||
const avatar = qu.q('.itemimg img');
|
||||
const avatar = query.element('.itemimg img');
|
||||
actor.avatar = `${site.url}/${avatar.src}`;
|
||||
actor.name = avatar.alt
|
||||
.split(':').slice(-1)[0]
|
||||
.replace(/xtreme girl|nurse/ig, '')
|
||||
.trim();
|
||||
|
||||
const actorPath = qu.url('.itemimg');
|
||||
const actorPath = query.attribute('.itemimg', 'href', raw);
|
||||
actor.url = `${site.url}${actorPath.slice(1)}`;
|
||||
|
||||
return actor;
|
||||
@@ -96,11 +99,11 @@ function extractModels({ el }, site) {
|
||||
|
||||
async function fetchModels(site, page = 1, accModels = []) {
|
||||
const url = `${site.url}/?models/${page}`;
|
||||
const res = await get(url);
|
||||
const res = await unprint.get(url);
|
||||
|
||||
if (res.ok) {
|
||||
const models = extractModels(res.item, site);
|
||||
const nextPage = res.item.qa('.pagenumbers', true)
|
||||
const models = extractModels(res.context, site);
|
||||
const nextPage = res.context.query.contents('.pagenumbers')
|
||||
.map((pageX) => Number(pageX))
|
||||
.filter(Boolean) // remove << and >>
|
||||
.includes(page + 1);
|
||||
@@ -117,16 +120,16 @@ async function fetchModels(site, page = 1, accModels = []) {
|
||||
|
||||
async function fetchLatest(site, page = 1, include, { beforeFetchLatest }) {
|
||||
const url = `${site.url}/show.php?a=${site.parameters.a}_${page}`;
|
||||
const res = await geta(url, '.item');
|
||||
const res = await unprint.get(url, { selectAll: '.item' });
|
||||
|
||||
return res.ok ? scrapeLatest(res.items, site, beforeFetchLatest) : res.status;
|
||||
return res.ok ? scrapeLatest(res.context, site, beforeFetchLatest) : res.status;
|
||||
}
|
||||
|
||||
async function fetchScene(url, site, release, include, beforeFetchLatest) {
|
||||
const models = beforeFetchLatest || await fetchModels(site);
|
||||
const res = await get(url);
|
||||
const res = await unprint.get(url);
|
||||
|
||||
return res.ok ? scrapeScene(res.item, url, site, models) : res.status;
|
||||
return res.ok ? scrapeScene(res.context, url, site, include, models) : res.status;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
|
||||
+38
-21
@@ -6,11 +6,13 @@ const { decode } = require('html-entities');
|
||||
const slugify = require('../utils/slugify');
|
||||
const { feetInchesToCm, femaleFeetUsToEu } = require('../utils/convert');
|
||||
|
||||
const tagKeys = ['tags', 'category', 'categories'];
|
||||
|
||||
function scrapeScene(data, channel, parameters) {
|
||||
const release = {};
|
||||
|
||||
release.entryId = data.cms_set_id;
|
||||
release.url = `${parameters.scene || channel.url}/video/${data.cms_set_id}`;
|
||||
release.url = `${parameters.scene || channel.url}${parameters.scenePath || '/video'}/${parameters.entryIdSlug ? data.slug : data.cms_set_id}`;
|
||||
|
||||
release.title = data.name;
|
||||
release.description = data.description;
|
||||
@@ -20,7 +22,7 @@ function scrapeScene(data, channel, parameters) {
|
||||
|
||||
release.actors = data.data_types.find((dataType) => dataType.data_type === 'Models' || dataType.data_type === 'Talent')?.data_values.map((actor) => ({
|
||||
name: actor.name,
|
||||
url: `${channel.url}/model/${actor.cms_data_value_id}?models=${encodeURI(actor.name)}`, // slug does not work unless it's also the ID
|
||||
url: `${channel.url}${parameters.modelPath || '/model'}/${parameters.entryIdSlug && actor.slug ? actor.slug : actor.cms_data_value_id}?models=${encodeURI(actor.name)}`, // slug does not work unless it's also the ID
|
||||
}));
|
||||
|
||||
release.directors = data.data_types.find((dataType) => dataType.data_type === 'Videographers')?.data_values.map((director) => ({
|
||||
@@ -29,7 +31,7 @@ function scrapeScene(data, channel, parameters) {
|
||||
}));
|
||||
|
||||
release.tags = data.data_types
|
||||
.filter((dataType) => dataType.data_type === 'Tags' || dataType.data_type === 'Category')
|
||||
.filter((dataType) => tagKeys.includes(dataType.data_type?.toLowerCase()))
|
||||
.flatMap((tags) => tags.data_values.map((tag) => tag.name));
|
||||
|
||||
const poster = data.preview_formatted.thumb;
|
||||
@@ -43,7 +45,7 @@ function scrapeScene(data, channel, parameters) {
|
||||
.map((key) => unprint.prefixUrl(`${poster[key][0].fileuri}?${poster[key][0].signature}`, parameters.cdn));
|
||||
}
|
||||
|
||||
release.caps = data.content
|
||||
release[parameters.imageType || 'caps'] = data.content
|
||||
?.filter((item) => item.content_type === 'image' && item.content?.[0])
|
||||
.map((item) => [
|
||||
item.content[0],
|
||||
@@ -66,13 +68,14 @@ function scrapeScene(data, channel, parameters) {
|
||||
release.trailer = unprint.prefixUrl(`${trailer.fileuri}?${trailer.signature}`, parameters.cdn);
|
||||
}
|
||||
|
||||
release.channel = slugify(data.data_types.find((dataType) => dataType.data_type === 'Series')?.data_values[0]?.name, '');
|
||||
release.channel = slugify(data.data_types.find((dataType) => dataType.data_type === 'Series')?.data_values[0]?.name, '') || null;
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
async function getBlockId(slug, dataSource, entity, parameters) {
|
||||
const res = await unprint.get(`https://azianistudios.com/tour_api.php/content/page?slug=${slug}&data_source=${JSON.stringify(dataSource)}`, {
|
||||
const res = await unprint.get(`${parameters.api || 'https://azianistudios.com/tour_api.php'}/content/page?slug=${slug}&data_source=${JSON.stringify(dataSource)}`, {
|
||||
interface: 'request', // necessary for KSM/Cosplayground CF wall, ok for Aziani
|
||||
headers: {
|
||||
Referer: entity.url,
|
||||
'x-nats-cms-area-id': parameters.areaId,
|
||||
@@ -109,9 +112,10 @@ async function fetchLatest(channel, page = 1, { parameters }) {
|
||||
data_type_search: parameters.seriesId && JSON.stringify({ 7: parameters.seriesId.toString() }), // doesn't seem relevant
|
||||
}).toString();
|
||||
|
||||
const url = `https://azianistudios.com/tour_api.php/content/sets?${query}`;
|
||||
const url = `${parameters.api || 'https://azianistudios.com/tour_api.php'}/content/sets?${query}`;
|
||||
|
||||
const res = await unprint.get(url, {
|
||||
interface: 'request', // necessary for KSM/Cosplayground CF wall, ok for Aziani
|
||||
headers: {
|
||||
Referer: channel.url,
|
||||
'x-nats-cms-area-id': parameters.areaId,
|
||||
@@ -126,20 +130,21 @@ async function fetchLatest(channel, page = 1, { parameters }) {
|
||||
}
|
||||
|
||||
async function fetchScene(url, entity, _baseRelease, { parameters }) {
|
||||
const entryId = new URL(url).pathname.match(/\/video\/(\w+)/)[1];
|
||||
const entryId = new URL(url).pathname.match(/\/videos?\/([\w-]+)/)[1];
|
||||
|
||||
if (!entryId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const blockId = await getBlockId('/video/:id', entryId, entity, parameters);
|
||||
const blockId = await getBlockId(`${parameters.videos || '/video'}/${parameters.entryIdSlug ? ':slug' : ':id'}`, entryId, entity, parameters);
|
||||
|
||||
if (!blockId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const query = new URLSearchParams({
|
||||
cms_set_ids: entryId,
|
||||
cms_set_ids: parameters.entryIdSlug ? '' : entryId,
|
||||
...parameters.entryIdSlug ? { slug: entryId } : null,
|
||||
cms_area_id: parameters.areaId,
|
||||
cms_block_id: blockId,
|
||||
content: 1,
|
||||
@@ -150,9 +155,10 @@ async function fetchScene(url, entity, _baseRelease, { parameters }) {
|
||||
content_count: 1,
|
||||
}).toString();
|
||||
|
||||
const apiUrl = `https://azianistudios.com/tour_api.php/content/sets?${query}`;
|
||||
const apiUrl = `${parameters.api || 'https://azianistudios.com/tour_api.php'}/content/sets?${query}`;
|
||||
|
||||
const res = await unprint.get(apiUrl, {
|
||||
interface: 'request', // necessary for KSM/Cosplayground CF wall, ok for Aziani
|
||||
headers: {
|
||||
Referer: entity.url,
|
||||
'x-nats-cms-area-id': parameters.areaId,
|
||||
@@ -174,14 +180,18 @@ function scrapeProfile(data, entity, parameters) {
|
||||
decode(detail.value || detail.content_formatted || detail.content),
|
||||
]));
|
||||
|
||||
profile.url = `${entity.url}/model/${data.cms_data_value_id}`;
|
||||
profile.url = `${entity.url}/model/${parameters.entryIdSlug ? data.slug : data.cms_data_value_id}`;
|
||||
profile.entryId = data.cms_data_value_id;
|
||||
|
||||
profile.description = data.description;
|
||||
|
||||
profile.gender = bio.gender?.toLowerCase();
|
||||
profile.age = bio.age;
|
||||
profile.dateOfBirth = unprint.extractDate(`${bio.born} 0`, 'MMMM Do YYYY', { match: /\w+ \d+\w{2} \d{1,4}/ });
|
||||
profile.dateOfBirth = unprint.extractDate(`${bio.born} 0`, 'MMMM Do YYYY', { match: /\w+ \d+\w{2} \d{1,4}/ })
|
||||
|| unprint.extractDate(bio.date_of_birth, 'yyyy-MM-dd');
|
||||
|
||||
profile.birthPlace = bio.birthplace;
|
||||
profile.ethnicity = bio.ethnicity;
|
||||
|
||||
profile.measurements = bio.measurements;
|
||||
profile.height = feetInchesToCm(bio.height);
|
||||
@@ -190,7 +200,9 @@ function scrapeProfile(data, entity, parameters) {
|
||||
profile.hairColor = bio.hair_color;
|
||||
profile.eyeColor = bio.eye_color;
|
||||
|
||||
const avatar = bio.thumbnail?.image[0];
|
||||
const avatar = bio.thumbnail?.image?.[0] // aziani
|
||||
|| bio.biopic?.image?.[0] // cosplayground profile
|
||||
|| bio.headshot?.image?.[0]; // cosplayground headshot
|
||||
|
||||
if (avatar) {
|
||||
profile.avatar = {
|
||||
@@ -201,36 +213,41 @@ function scrapeProfile(data, entity, parameters) {
|
||||
};
|
||||
}
|
||||
|
||||
profile.socials = [bio.onlyfans, bio.instagram].filter(Boolean);
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchProfile({ name, url }, { entity, parameters }) {
|
||||
if (!url) {
|
||||
// no easy search option
|
||||
if (!url && !parameters.entryIdSlug) {
|
||||
// no easy search option and numeric ID is non-derivable
|
||||
return null;
|
||||
}
|
||||
|
||||
const actorId = new URL(url).pathname.match(/model\/(\d+)/)[1];
|
||||
const actorId = url && new URL(url).pathname.match(/models?\/(\d+)/)?.[1];
|
||||
|
||||
if (!actorId) {
|
||||
if (!actorId && !parameters.entryIdSlug) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const blockId = await getBlockId('/model/:id', { models: name, id: actorId }, entity, parameters);
|
||||
const slug = slugify(name);
|
||||
const blockId = await getBlockId(`${parameters.modelPath || '/model'}/${parameters.entryIdSlug ? ':slug' : 'id'}`, { models: name, id: actorId, slug }, entity, parameters);
|
||||
|
||||
if (!blockId) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const query = new URLSearchParams({
|
||||
cms_data_value_ids: actorId,
|
||||
cms_data_value_ids: actorId || '',
|
||||
...parameters.entryIdSlug ? { slug } : null,
|
||||
cms_block_id: blockId,
|
||||
cms_data_type_id: 4,
|
||||
}).toString();
|
||||
|
||||
const apiUrl = `https://azianistudios.com/tour_api.php/content/data-values?${query}`;
|
||||
const apiUrl = `${parameters.api || 'https://azianistudios.com/tour_api.php'}/content/data-values?${query}`;
|
||||
|
||||
const res = await unprint.get(apiUrl, {
|
||||
interface: 'request', // necessary for KSM/Cosplayground CF wall, ok for Aziani
|
||||
headers: {
|
||||
Referer: entity.url,
|
||||
'x-nats-cms-area-id': entity.parameters.areaId,
|
||||
|
||||
+35
-31
@@ -1,6 +1,7 @@
|
||||
'use strict';
|
||||
|
||||
const qu = require('../utils/qu');
|
||||
const unprint = require('unprint');
|
||||
|
||||
const slugify = require('../utils/slugify');
|
||||
const { convert } = require('../utils/convert');
|
||||
|
||||
@@ -21,23 +22,23 @@ function scrapeAll(scenes, channel) {
|
||||
return scenes.map(({ query }) => {
|
||||
const release = {};
|
||||
|
||||
release.url = query.url('a', 'href', { origin: channel.url });
|
||||
release.entryId = query.q(null, 'data-video-card-scene-id') || query.q('video', 'data-video-preview-scene-id') || new URL(release.url).pathname.match(/(\d+)\/?$/)?.[1];
|
||||
release.url = query.url('a', { origin: channel.url });
|
||||
release.entryId = query.attribute(null, 'data-video-card-scene-id') || query.attribute('video', 'data-video-preview-scene-id') || new URL(release.url).pathname.match(/(\d+)\/?$/)?.[1];
|
||||
|
||||
release.title = query.cnt('.video-card-title span') || query.q('.video-card-title', 'title');
|
||||
release.description = query.cnt('.video-card-description');
|
||||
release.title = query.content('.video-card-title span') || query.attribute('.video-card-title', 'title');
|
||||
release.description = query.content('.video-card-description');
|
||||
|
||||
release.date = query.date('.video-card-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-card-upload-date', 'MMMM DD, YYYY');
|
||||
release.duration = query.duration('.video-card-duration', null, 'content') || query.number('.video-card-duration') * 60;
|
||||
release.date = query.date('.video-card-upload-date', 'YYYY-MM-DD HH:mm:ss', { attribute: 'content' }) || query.date('.video-card-upload-date', 'MMMM DD, YYYY');
|
||||
release.duration = query.duration('.video-card-duration', { attribute: 'content' }) || query.number('.video-card-duration') * 60;
|
||||
|
||||
release.actors = query.all('.video-card-details--cast a').map((el) => ({
|
||||
name: qu.query.cnt(el),
|
||||
url: qu.query.url(el, null, 'href', { origin: channel.url }),
|
||||
name: unprint.query.content(el),
|
||||
url: unprint.query.url(el, null, { origin: channel.url }),
|
||||
}));
|
||||
|
||||
release.tags = query.cnts('.video-card-tags a');
|
||||
release.tags = query.contents('.video-card-tags a');
|
||||
|
||||
const posterSources = query.srcset('picture source', 'data-srcset') || [query.img('.video-card-image', 'data-src')];
|
||||
const posterSources = query.sourceSet('picture source', 'data-srcset') || [query.img('.video-card-image', { attribute: 'data-src' })];
|
||||
release.poster = getPoster(posterSources);
|
||||
|
||||
release.teaser = query.video('video') || (channel.parameters?.teasers !== false ? `https://p.badoinkvr.com/d/scene_files/${release.entryId}/preview/preview.mp4` : null);
|
||||
@@ -46,29 +47,29 @@ function scrapeAll(scenes, channel) {
|
||||
});
|
||||
}
|
||||
|
||||
function scrapeScene({ query }, url, channel) {
|
||||
function scrapeScene({ query }, { url, entity: channel }) {
|
||||
const release = {};
|
||||
|
||||
release.entryId = new URL(url).pathname.match(/(\d+)\/?$/)?.[1];
|
||||
|
||||
release.title = query.cnt('.video-title', '.breadcrumbs__item:last-child [itemprop=name]');
|
||||
release.description = query.cnt('.video-description');
|
||||
release.title = query.content(['.video-title', '.breadcrumbs__item:last-child [itemprop=name]']);
|
||||
release.description = query.content('.video-description');
|
||||
|
||||
release.date = query.date('.video-upload-date', 'YYYY-MM-DD HH:mm:ss', null, 'content') || query.date('.video-upload-date', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
|
||||
release.duration = query.duration('.video-duration', null, 'content') || query.number('.video-duration') * 60;
|
||||
release.date = query.date('.video-upload-date', 'YYYY-MM-DD HH:mm:ss', { attribute: 'content' }) || query.date('.video-upload-date', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
|
||||
release.duration = query.duration('.video-duration', { attribute: 'content' }) || query.number('.video-duration') * 60;
|
||||
|
||||
release.actors = query.all('.video-actors a').map((el) => ({
|
||||
name: qu.query.cnt(el),
|
||||
url: qu.query.url(el, null, 'href', { origin: channel.url }),
|
||||
name: unprint.query.content(el),
|
||||
url: unprint.query.url(el, null, { origin: channel.url }),
|
||||
}));
|
||||
|
||||
release.tags = query.cnts('.video-tags a');
|
||||
release.tags = query.contents('.video-tags a');
|
||||
|
||||
const posterSources = query.srcset('.video-image-container picture source', 'srcset') || [query.img('.video-image')];
|
||||
const posterSources = query.sourceSet('.video-image-container picture source', 'srcset') || [query.img('.video-image')];
|
||||
release.poster = getPoster(posterSources);
|
||||
|
||||
const photos = query.imgs('.gallery .gallery-item', 'data-big-image');
|
||||
const extraPhotoCount = query.number('.gallery .gallery-item:last-child', /\d+/, 'data-desktop-text');
|
||||
const photos = query.imgs('.gallery .gallery-item', { attribute: 'data-big-image' });
|
||||
const extraPhotoCount = query.number('.gallery .gallery-item:last-child', { match: /\d+/, attribute: 'data-desktop-text' });
|
||||
const lastPhoto = photos.slice(-1)[0];
|
||||
const photosHaveIndex = /_\d+\./.test(lastPhoto);
|
||||
|
||||
@@ -87,10 +88,10 @@ function scrapeProfile({ query }, url, entity) {
|
||||
|
||||
const bio = query.all('.girl-details-stats-item').reduce((acc, el) => ({
|
||||
...acc,
|
||||
[slugify(qu.query.cnt(el, '.girl-details-stat'))]: qu.query.cnt(el, '.girl-details-stat-value'),
|
||||
[slugify(unprint.query.content(el, '.girl-details-stat'))]: unprint.query.content(el, '.girl-details-stat-value'),
|
||||
}), {});
|
||||
|
||||
profile.description = query.cnt('.girl-details-bio');
|
||||
profile.description = query.content('.girl-details-bio');
|
||||
|
||||
profile.age = bio.age;
|
||||
profile.birthPlace = bio.country;
|
||||
@@ -104,23 +105,26 @@ function scrapeProfile({ query }, url, entity) {
|
||||
profile.hairColor = bio.hair;
|
||||
profile.eyes = bio.eyes;
|
||||
|
||||
const avatarSources = query.srcset('.girl-details-photo-content picture source', 'srcset') || [query.img('.girl-details-photo')];
|
||||
const avatarSources = query.sourceSet('.girl-details-photo-content picture source', 'srcset') || [query.img('.girl-details-photo')];
|
||||
|
||||
profile.avatar = getPoster(avatarSources).slice(0, 3); // returns a metric ton of links, if first few don't work the rest presumably won't either
|
||||
profile.social = query.urls('.girl-details-social-media-list a');
|
||||
|
||||
profile.scenes = scrapeAll(qu.initAll(query.all('.video-card')), entity);
|
||||
profile.scenes = scrapeAll(unprint.initAll(query.all('.video-card')), entity);
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page) {
|
||||
const res = await qu.getAll(`${channel.url}/${channel.parameters?.latest || 'vrpornvideos'}/${page}`, '.video-card', {
|
||||
Cookie: 'affsubid=12345-;', // required to show teaser video, exact number doesn't seem to matter
|
||||
const res = await unprint.get(`${channel.url}/${channel.parameters?.latest || 'vrpornvideos'}/${page}`, {
|
||||
selectAll: '.video-card',
|
||||
cookies: {
|
||||
affsubid: '12345-', // required to show teaser video, exact number doesn't seem to matter
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeAll(res.items, channel);
|
||||
return scrapeAll(res.context, channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
@@ -128,10 +132,10 @@ async function fetchLatest(channel, page) {
|
||||
|
||||
async function fetchProfile(baseActor, { entity }) {
|
||||
const url = `${entity.url}/${entity.parameters?.actor || 'pornstar'}/${slugify(baseActor.name, '-')}/`;
|
||||
const res = await qu.get(url);
|
||||
const res = await unprint.get(url);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeProfile(res.item, url, entity);
|
||||
return scrapeProfile(res.context, url, entity);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
|
||||
+17
-17
@@ -1,29 +1,29 @@
|
||||
'use strict';
|
||||
|
||||
const qu = require('../utils/q');
|
||||
const unprint = require('unprint');
|
||||
|
||||
function scrapeAll(scenes, channel) {
|
||||
return scenes.map(({ query }) => {
|
||||
const release = {};
|
||||
|
||||
release.url = query.url('a', 'href', { origin: channel.url });
|
||||
release.url = query.url('a', { origin: channel.url });
|
||||
release.entryId = new URL(release.url).pathname.match(/\/tour\/preview\/(.+)/)[1];
|
||||
|
||||
release.title = query.cnt('.nice-title');
|
||||
release.title = query.content('.nice-title');
|
||||
|
||||
release.poster = query.img('img.poster');
|
||||
release.photos = query.q('img.poster', 'data-urls')?.split(' ');
|
||||
release.teaser = query.video('img.poster', 'data-thumbnail');
|
||||
release.photos = query.attribute('img.poster', 'data-urls')?.split(' ');
|
||||
release.teaser = query.video('img.poster', { attribute: 'data-thumbnail' });
|
||||
|
||||
if (query.exists('.info')) {
|
||||
release.date = query.date('.updated', 'MM/DD/YYYY');
|
||||
|
||||
release.actors = [{
|
||||
name: query.cnt('.name'),
|
||||
age: query.cnt('.age'),
|
||||
name: query.content('.name'),
|
||||
age: query.content('.age'),
|
||||
}];
|
||||
|
||||
release.tags = query.cnts('.tags a');
|
||||
release.tags = query.contents('.tags a');
|
||||
}
|
||||
|
||||
return release;
|
||||
@@ -35,38 +35,38 @@ function scrapeScene({ query }, url) {
|
||||
|
||||
release.entryId = new URL(url).pathname.match(/\/tour\/preview\/(.+)/)[1];
|
||||
|
||||
release.title = query.cnt('.nice-title');
|
||||
release.description = query.cnt('.desc-text p');
|
||||
release.title = query.content('.nice-title');
|
||||
release.description = query.content('.desc-text p');
|
||||
|
||||
release.tags = query.cnts('.tags a');
|
||||
release.tags = query.contents('.tags a');
|
||||
|
||||
release.poster = query.poster();
|
||||
release.photos = query.imgs('.thumbnails img, .thumb img');
|
||||
|
||||
// some videos may not have unique trailers, but many do
|
||||
release.trailer = query.video();
|
||||
release.trailer = query.video('source');
|
||||
|
||||
release.formats = query.cnts('.download-quality option');
|
||||
release.formats = query.contents('.download-quality option');
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page = 1) {
|
||||
const url = `${channel.url}/tour/videos/page-${page}`;
|
||||
const res = await qu.getAll(url, '.episode-list .episode, .content .girl');
|
||||
const res = await unprint.get(url, { selectAll: '.episode-list .episode, .content .girl' });
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeAll(res.items, channel);
|
||||
return scrapeAll(res.context, channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchScene(url, channel) {
|
||||
const res = await qu.get(url);
|
||||
const res = await unprint.get(url);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeScene(res.item, url, channel);
|
||||
return scrapeScene(res.context, url, channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
'use strict';
|
||||
|
||||
const unprint = require('unprint');
|
||||
const cookie = require('cookie');
|
||||
|
||||
const slugify = require('../utils/slugify');
|
||||
|
||||
@@ -54,16 +53,14 @@ function scrapeAll(scenes, channel) {
|
||||
|
||||
async function beforeFetchLatest(channel) {
|
||||
// scene page only seems to accept language preferences from session
|
||||
const { res } = await unprint.get(`${channel.url}/en/news-videos-x-marc-dorcel`, {
|
||||
const { cookies } = await unprint.get(`${channel.url}/en/news-videos-x-marc-dorcel`, {
|
||||
headers: {
|
||||
'X-Requested-With': 'XMLHttpRequest',
|
||||
'Accept-Language': 'en-US,en', // fetch English rather than French titles
|
||||
},
|
||||
});
|
||||
|
||||
const sessionCookie = cookie.parse(res.headers['set-cookie'][0])?.dorcelclub;
|
||||
|
||||
return `dorcelclub=${sessionCookie}`;
|
||||
return `dorcelclub=${cookies.dorcelclub}`;
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page = 1, _options, { beforeFetchLatest: sessionCookie }) {
|
||||
|
||||
+50
-51
@@ -1,8 +1,8 @@
|
||||
'use strict';
|
||||
|
||||
const format = require('template-format');
|
||||
const unprint = require('unprint');
|
||||
|
||||
const qu = require('../utils/q');
|
||||
const slugify = require('../utils/slugify');
|
||||
const { convert } = require('../utils/convert');
|
||||
|
||||
@@ -10,11 +10,11 @@ function deriveEntryId(release) {
|
||||
if (release.date && release.url) {
|
||||
const slug = new URL(release.url).pathname.match(/\/trailers\/(.*).html/)[1];
|
||||
|
||||
return `${slugify(qu.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(slug)}`;
|
||||
return `${slugify(unprint.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(slug)}`;
|
||||
}
|
||||
|
||||
if (release.date && release.title) {
|
||||
return `${slugify(qu.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
|
||||
return `${slugify(unprint.formatDate(release.date, 'YYYY-MM-DD'))}-${slugify(release.title)}`;
|
||||
}
|
||||
|
||||
return null;
|
||||
@@ -22,7 +22,7 @@ function deriveEntryId(release) {
|
||||
|
||||
function extractPoster(posterPath, channel, baseRelease) {
|
||||
if (posterPath && !/400.jpg/.test(posterPath)) {
|
||||
const poster = qu.prefixUrl(posterPath, channel.parameters?.media || channel.url);
|
||||
const poster = unprint.prefixUrl(posterPath, channel.parameters?.media || channel.url);
|
||||
const posterSources = [
|
||||
poster,
|
||||
// upscaled
|
||||
@@ -40,20 +40,20 @@ function extractPoster(posterPath, channel, baseRelease) {
|
||||
return [baseRelease?.poster || null, []];
|
||||
}
|
||||
|
||||
function getImageWithFallbacks(q, selector, site, el) {
|
||||
function getImageWithFallbacks(query, selector, site, el) {
|
||||
const sources = el
|
||||
? [
|
||||
q(el, selector, 'src0_3x'),
|
||||
q(el, selector, 'src0_2x'),
|
||||
q(el, selector, 'src0_1x'),
|
||||
unprint.query.attribute(el, selector, 'src0_3x'),
|
||||
unprint.query.attribute(el, selector, 'src0_2x'),
|
||||
unprint.query.attribute(el, selector, 'src0_1x'),
|
||||
]
|
||||
: [
|
||||
q(selector, 'src0_3x'),
|
||||
q(selector, 'src0_2x'),
|
||||
q(selector, 'src0_1x'),
|
||||
query.attribute(selector, 'src0_3x'),
|
||||
query.attribute(selector, 'src0_2x'),
|
||||
query.attribute(selector, 'src0_1x'),
|
||||
];
|
||||
|
||||
return sources.filter(Boolean).map((src) => qu.prefixUrl(src, site.parameters?.media || site.url));
|
||||
return sources.filter(Boolean).map((src) => unprint.prefixUrl(src, site.parameters?.media || site.url));
|
||||
}
|
||||
|
||||
function scrapeAllClassic(scenes, channel) {
|
||||
@@ -61,16 +61,16 @@ function scrapeAllClassic(scenes, channel) {
|
||||
const release = {};
|
||||
|
||||
release.url = query.url('.updateInfo h5 a:not([href*="content/"]):not([href*="#coming"])');
|
||||
release.entryId = query.url('.updateThumb img', 'alt');
|
||||
release.entryId = query.attribute('.updateThumb img', 'alt');
|
||||
|
||||
release.title = query.cnt('.updateInfo h5 a');
|
||||
release.title = query.content('.updateInfo h5 a');
|
||||
|
||||
release.actors = query.cnts('.tour_update_models a');
|
||||
release.actors = query.contents('.tour_update_models a');
|
||||
release.date = query.date('.availdate, .updateInfo p span:last-child', 'MM/DD/YYYY');
|
||||
|
||||
release.poster = query.img('.updateThumb img');
|
||||
release.poster = query.img('.updateThumb img', { origin: channel.url }); // relative to site root, not listing page
|
||||
|
||||
const trailer = query.q('.updateInfo h5 a', 'onclick')?.match(/'(.+)'/)?.[1];
|
||||
const trailer = query.attribute('.updateInfo h5 a', 'onclick')?.match(/'(.+)'/)?.[1];
|
||||
|
||||
if (trailer) {
|
||||
release.trailer = `${channel.url}${trailer}`;
|
||||
@@ -84,13 +84,14 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
|
||||
return scenes.map(({ query }) => {
|
||||
const release = {};
|
||||
|
||||
release.title = query.q('h4 a', 'title') || query.q('h4 a', true);
|
||||
release.title = query.attribute('h4 a', 'title') || query.content('h4 a');
|
||||
release.url = query.url('h4 a');
|
||||
|
||||
release.date = query.date('.more-info-div', 'MMM D, YYYY');
|
||||
release.duration = query.dur('.more-info-div');
|
||||
release.duration = query.duration('.more-info-div');
|
||||
|
||||
const posterPath = query.q('.img-div img', 'src0_1x') || query.img('img.video_placeholder');
|
||||
// raw path, prefixed with media URL below
|
||||
const posterPath = query.attribute('.img-div img', 'src0_1x') || query.attribute('img.video_placeholder', 'data-src') || query.attribute('img.video_placeholder', 'src', { forceGetAttribute: true });
|
||||
|
||||
if (posterPath) {
|
||||
const poster = /^http/.test(posterPath) ? posterPath : `${channel.parameters?.media || channel.url}${posterPath}`;
|
||||
@@ -102,7 +103,7 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
|
||||
];
|
||||
}
|
||||
|
||||
release.teaser = query.video();
|
||||
release.teaser = query.video('source');
|
||||
|
||||
// release.entryId = q('.img-div img', 'id')?.match(/set-target-(\d+)/)[1];
|
||||
release.entryId = deriveEntryId(release);
|
||||
@@ -116,11 +117,11 @@ function scrapeAllTubular(scenes, channel, accNetworkReleases) {
|
||||
});
|
||||
}
|
||||
|
||||
function scrapeSceneClassic({ query, html }, url, channel) {
|
||||
function scrapeSceneClassic({ query, html }, { entity: channel }) {
|
||||
const release = {};
|
||||
|
||||
release.title = query.q('.updatesBlock h2', true);
|
||||
release.poster = query.meta('property="og:image"');
|
||||
release.title = query.content('.updatesBlock h2');
|
||||
release.poster = query.attribute('meta[property="og:image"]', 'content');
|
||||
release.entryId = release.poster.match(/\/content\/(.*)\//)?.[1];
|
||||
|
||||
const trailer = html.match(/src="(.+\.mp4)"/)?.[1];
|
||||
@@ -134,34 +135,34 @@ function scrapeSceneClassic({ query, html }, url, channel) {
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
|
||||
function scrapeSceneTubular({ query, html }, { entity, url, baseRelease }) {
|
||||
const release = {};
|
||||
|
||||
release.title = query.q('.video-player .section-title', true) || query.q('.bio-section-head .section-title', true);
|
||||
release.title = query.content('.video-player .section-title') || query.content('.bio-section-head .section-title');
|
||||
release.description = query.text('.update-info-block .descriptionFull') || query.text('.update-info-block .description');
|
||||
|
||||
release.date = query.date('.update-info-row', 'MMM D, YYYY', /\w+ \d{1,2}, \d{4}/);
|
||||
release.duration = query.dur('.update-info-row:nth-child(2)');
|
||||
release.photoCount = query.number('.update-info-row:nth-child(2)', /(\d+) photos/i, 'textContent', 1);
|
||||
release.date = query.date('.update-info-row', 'MMM D, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
|
||||
release.duration = query.duration('.update-info-row:nth-child(2)');
|
||||
release.photoCount = query.number('.update-info-row:nth-child(2)', { match: /(\d+) photos/i, matchIndex: 1 });
|
||||
|
||||
release.actors = query.all('.models-list-thumbs a').map((el) => ({
|
||||
name: query.cnt(el, 'span'),
|
||||
avatar: getImageWithFallbacks(query.q, 'img', entity, el),
|
||||
url: query.url(el, null),
|
||||
name: unprint.query.content(el, 'span'),
|
||||
avatar: getImageWithFallbacks(query, 'img', entity, el),
|
||||
url: unprint.query.url(el, null),
|
||||
}));
|
||||
|
||||
release.tags = query.all('.tags a', true);
|
||||
release.tags = query.contents('.tags a');
|
||||
|
||||
const posterPath = query.q('.player-thumb img', 'src0_1x');
|
||||
const posterPath = query.attribute('.player-thumb img', 'src0_1x');
|
||||
const trailer = html.match(/<video.*src="(.*\.mp4)/)?.[1];
|
||||
|
||||
[release.poster, release.photos] = extractPoster(posterPath, entity, baseRelease);
|
||||
|
||||
if (trailer) {
|
||||
release.trailer = { src: qu.prefixUrl(trailer, entity.parameters?.media || entity.url), referer: url };
|
||||
release.trailer = { src: unprint.prefixUrl(trailer, entity.parameters?.media || entity.url), referer: url };
|
||||
}
|
||||
|
||||
const stars = query.q('.update-rating', true).match(/\d.\d/)?.[0];
|
||||
const stars = query.content('.update-rating').match(/\d.\d/)?.[0];
|
||||
if (stars) release.stars = Number(stars);
|
||||
|
||||
if (entity.type === 'network') {
|
||||
@@ -181,11 +182,11 @@ function scrapeSceneTubular({ query, html }, entity, url, baseRelease) {
|
||||
async function scrapeProfile({ query }, entity, parameters) {
|
||||
const profile = {};
|
||||
|
||||
const bio = query.cnt('.model_bio, .detail-div');
|
||||
const avatarEl = query.q('.model_bio_pic img, .model_bio_thumb');
|
||||
const bio = query.content('.model_bio, .detail-div');
|
||||
const avatarEl = query.element('.model_bio_pic img, .model_bio_thumb');
|
||||
|
||||
profile.age = Number(bio?.match(/Age:\s*(\d{2})/)?.[1]) || null;
|
||||
profile.dateOfBirth = qu.parseDate(bio?.match(/Age:\s*(\w+ \d{1,2}, \d{4})/)?.[0], 'MMMM D, YYYY');
|
||||
profile.dateOfBirth = unprint.extractDate(bio?.match(/Age:\s*(\w+ \d{1,2}, \d{4})/)?.[0], 'MMMM D, YYYY');
|
||||
|
||||
profile.height = convert(bio?.match(/\d+\s*(feet|')\s*\d+\s*(inches|"|$)/)?.[0], 'cm');
|
||||
profile.measurements = bio?.match(/\w+[-x]\d+[-x]\d+/)?.[0] || null;
|
||||
@@ -201,17 +202,17 @@ async function scrapeProfile({ query }, entity, parameters) {
|
||||
avatarEl.getAttribute('src'),
|
||||
]
|
||||
.filter((avatar) => avatar && !/p\d+.jpe?g/.test(avatar)) // remove non-existing attributes and placeholder images
|
||||
.map((avatar) => qu.prefixUrl(avatar, entity.url));
|
||||
.map((avatar) => unprint.prefixUrl(avatar, entity.url));
|
||||
|
||||
if (avatarSources.length) profile.avatar = avatarSources;
|
||||
}
|
||||
|
||||
if (parameters?.layout === 'classic') {
|
||||
profile.scenes = scrapeAllClassic(qu.initAll(query.all('.bodyArea .updateItem')), entity);
|
||||
profile.scenes = scrapeAllClassic(unprint.initAll(query.all('.bodyArea .updateItem')), entity);
|
||||
}
|
||||
|
||||
if (parameters?.layout === 'tubular') {
|
||||
profile.scenes = scrapeAllTubular(qu.initAll(query.all('.modelfeature, .item-video')), entity);
|
||||
profile.scenes = scrapeAllTubular(unprint.initAll(query.all('.modelfeature, .item-video')), entity);
|
||||
}
|
||||
|
||||
return profile;
|
||||
@@ -221,20 +222,20 @@ async function fetchLatest(site, page = 1, options, preData, allScraper) {
|
||||
const url = (site.parameters?.latest && format(site.parameters.latest, { page }))
|
||||
|| `${site.url}/categories/movies_${page}_d.html`;
|
||||
|
||||
const res = await qu.getAll(url, '.modelfeature, .item-video, .bodyArea .updateItem');
|
||||
const res = await unprint.get(url, { selectAll: '.modelfeature, .item-video, .bodyArea .updateItem' });
|
||||
|
||||
if (!res.ok) {
|
||||
return res.status;
|
||||
}
|
||||
|
||||
return allScraper(res.items, site, preData?.uniqueReleases);
|
||||
return allScraper(res.context, site, preData?.uniqueReleases);
|
||||
}
|
||||
|
||||
async function fetchUpcomingClassic(channel) {
|
||||
const res = await qu.getAll(channel.url, '#owl-upcomingScenes .updateItem');
|
||||
const res = await unprint.get(channel.url, { selectAll: '#owl-upcomingScenes .updateItem' });
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeAllClassic(res.items, channel);
|
||||
return scrapeAllClassic(res.context, channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
@@ -273,10 +274,10 @@ async function fetchProfile({ name: actorName, url }, { entity, parameters }) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const res = await qu.get(profileUrl);
|
||||
const res = await unprint.get(profileUrl);
|
||||
|
||||
if (res.statusCode === 200) {
|
||||
return scrapeProfile(res.item, entity, parameters);
|
||||
if (res.status === 200) {
|
||||
return scrapeProfile(res.context, entity, parameters);
|
||||
}
|
||||
|
||||
return null;
|
||||
@@ -290,14 +291,12 @@ module.exports = {
|
||||
fetchProfile,
|
||||
scrapeAll: scrapeAllClassic,
|
||||
scrapeScene: scrapeSceneClassic,
|
||||
deprecated: true,
|
||||
},
|
||||
tubular: {
|
||||
fetchLatest: fetchLatestTubular,
|
||||
fetchProfile,
|
||||
scrapeAll: scrapeAllTubular,
|
||||
scrapeScene: scrapeSceneTubular,
|
||||
deprecated: true,
|
||||
},
|
||||
getImageWithFallbacks,
|
||||
};
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
'use strict';
|
||||
|
||||
const unprint = require('unprint');
|
||||
const { nanoid } = require('nanoid');
|
||||
|
||||
function scrapeAll(scenes, _channel) {
|
||||
return scenes.map(({ query }) => {
|
||||
@@ -40,6 +41,12 @@ async function fetchLatest(channel, page = 1) {
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function beforeFetchScenes(channel) {
|
||||
const res = await unprint.get(channel.url);
|
||||
|
||||
return { nats: res.cookies.nats || nanoid() };
|
||||
}
|
||||
|
||||
function scrapeScene({ query, html }, { url, entity, baseRelease }) {
|
||||
const release = {};
|
||||
|
||||
@@ -81,6 +88,20 @@ function scrapeScene({ query, html }, { url, entity, baseRelease }) {
|
||||
return release;
|
||||
}
|
||||
|
||||
async function fetchScene(url, entity, baseRelease, context) {
|
||||
const res = await unprint.get(url, {
|
||||
cookies: {
|
||||
nats: context.beforeFetchScenes?.nats,
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeScene(res.context, { url, entity, baseRelease });
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
function scrapeProfile({ query }) {
|
||||
const profile = {};
|
||||
|
||||
@@ -110,5 +131,6 @@ async function fetchProfile(actor, entity) {
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchProfile,
|
||||
scrapeScene,
|
||||
fetchScene,
|
||||
beforeFetchScenes,
|
||||
};
|
||||
|
||||
+44
-39
@@ -1,25 +1,25 @@
|
||||
'use strict';
|
||||
|
||||
const qu = require('../utils/qu');
|
||||
const http = require('../utils/http');
|
||||
const unprint = require('unprint');
|
||||
|
||||
const slugify = require('../utils/slugify');
|
||||
|
||||
function scrapeLatest(scenes, site) {
|
||||
return scenes.map(({ query }) => {
|
||||
const release = {};
|
||||
|
||||
release.url = query.url('figure a', 'href', { origin: site.parameters.latest });
|
||||
release.url = query.url('figure a', { origin: site.parameters.latest });
|
||||
|
||||
release.title = query.cnt('.has-text-weight-bold, .is-size-6');
|
||||
release.title = query.content('.has-text-weight-bold, .is-size-6');
|
||||
release.date = query.date('span.tag', 'YYYY-MM-DD');
|
||||
release.actors = query.cnts('a.tag');
|
||||
release.actors = query.contents('a.tag');
|
||||
|
||||
const cover = query.img('.image img');
|
||||
|
||||
release.poster = cover.replace('poster_noplay', 'trailer_noplay');
|
||||
release.covers = [cover];
|
||||
|
||||
release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title.split(/\s+/).slice(0, 5).join(' '))}`;
|
||||
release.entryId = `${unprint.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title.split(/\s+/).slice(0, 5).join(' '))}`;
|
||||
|
||||
return release;
|
||||
});
|
||||
@@ -30,11 +30,11 @@ function scrapeLatestLegacy(scenes, site) {
|
||||
// if (q('.articleTitleText')) return scrapeFirstLatest(ctx(el), site);
|
||||
const release = {};
|
||||
|
||||
const titleEl = query.q('.galleryTitleText, .articleTitleText');
|
||||
const titleEl = query.element('.galleryTitleText, .articleTitleText');
|
||||
const [title, ...actors] = titleEl.textContent.split('|');
|
||||
const date = query.date('.articlePostDateText td', 'MMM D, YYYY');
|
||||
|
||||
const url = query.url(titleEl, 'a');
|
||||
const url = unprint.query.attribute(titleEl, 'a', 'href');
|
||||
[release.entryId] = url.split('/').slice(-2);
|
||||
release.url = `${site.url}${url}`;
|
||||
|
||||
@@ -44,20 +44,20 @@ function scrapeLatestLegacy(scenes, site) {
|
||||
} else {
|
||||
// title should contain date instead, not applicable in brief mode
|
||||
release.title = title.slice(title.indexOf(':') + 1).trim();
|
||||
release.date = qu.ed(title.slice(0, title.indexOf(':')), 'MMM D, YYYY');
|
||||
release.date = unprint.extractDate(title.slice(0, title.indexOf(':')), 'MMM D, YYYY');
|
||||
}
|
||||
|
||||
release.actors = actors.map((actor) => actor.trim());
|
||||
|
||||
const description = query.q('.articleCopyText', true);
|
||||
const description = query.content('.articleCopyText');
|
||||
if (description) release.description = description.slice(0, description.lastIndexOf('('));
|
||||
|
||||
const duration = query.dur('.articleCopyText a:nth-child(2)');
|
||||
const duration = query.duration('.articleCopyText a:nth-child(2)');
|
||||
if (duration) release.duration = duration;
|
||||
|
||||
release.likes = parseInt(query.q('.articlePostDateText td:nth-child(3)', true), 10);
|
||||
release.likes = query.number('.articlePostDateText td:nth-child(3)');
|
||||
|
||||
const cover = query.img('a img');
|
||||
const cover = query.img('a img', { origin: site.url });
|
||||
|
||||
release.covers = [[
|
||||
cover.replace('_thumbnail', ''),
|
||||
@@ -68,25 +68,26 @@ function scrapeLatestLegacy(scenes, site) {
|
||||
});
|
||||
}
|
||||
|
||||
async function scrapeScene({ query }, url, channel, parameters, session) {
|
||||
async function scrapeScene({ query }, channel, include, cookies) {
|
||||
const release = {};
|
||||
|
||||
release.title = query.cnt('.columns div.is-size-5.has-text-weight-bold');
|
||||
release.description = query.cnt('.has-background-black-ter > div:nth-child(4)');
|
||||
release.title = query.content('.columns div.is-size-5.has-text-weight-bold');
|
||||
release.description = query.content('.has-background-black-ter > div:nth-child(4)');
|
||||
release.date = query.date('.has-text-white-ter span.tag', 'YYYY-MM-DD');
|
||||
|
||||
release.actors = query.cnts('.has-text-white-ter a.tag[href*="home.php"]');
|
||||
release.tags = query.cnts('.has-background-black-ter > div:nth-child(6) > span');
|
||||
release.actors = query.contents('.has-text-white-ter a.tag[href*="home.php"]');
|
||||
release.tags = query.contents('.has-background-black-ter > div:nth-child(6) > span');
|
||||
|
||||
release.poster = query.img('#videoPlayer, #iodvideo', 'poster');
|
||||
release.photos = Array.from(query.html('body > div:nth-child(6)').matchAll(/src="(http.*jpg)"/g), (match) => match[1]);
|
||||
release.poster = query.img('#videoPlayer, #iodvideo', { attribute: 'poster' });
|
||||
release.photos = Array.from((query.html('body > div:nth-child(6)') || '').matchAll(/src="(http.*jpg)"/g), (match) => match[1]);
|
||||
|
||||
release.entryId = `${qu.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
|
||||
release.entryId = `${unprint.formatDate(release.date, 'YYYY-MM-DD')}-${slugify(release.title)}`;
|
||||
|
||||
release.trailer = query.video();
|
||||
release.trailer = query.video('source');
|
||||
|
||||
if (!release.trailer && parameters.includeTrailers) {
|
||||
const trailerRes = await http.get(`${channel.url}/api/play-api.php`, { session });
|
||||
if (!release.trailer && include.includeTrailers) {
|
||||
// play API relies on the session cookie set by the scene page
|
||||
const trailerRes = await unprint.get(`${channel.url}/api/play-api.php`, { cookies, extract: false });
|
||||
|
||||
if (trailerRes.ok) {
|
||||
release.trailer = trailerRes.body;
|
||||
@@ -96,22 +97,22 @@ async function scrapeScene({ query }, url, channel, parameters, session) {
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeSceneLegacy({ query }, site) {
|
||||
function scrapeSceneLegacy({ query }, { entity: site }) {
|
||||
const release = {};
|
||||
|
||||
const titleEl = query.q('.articleTitleText');
|
||||
const titleEl = query.element('.articleTitleText');
|
||||
const [title, ...actors] = titleEl.textContent.split('|');
|
||||
|
||||
const url = query.url(titleEl, 'a');
|
||||
const url = unprint.query.attribute(titleEl, 'a', 'href');
|
||||
[release.entryId] = url.split('/').slice(-2);
|
||||
release.url = `${site.url}${url}`;
|
||||
|
||||
release.title = title.trim();
|
||||
release.description = query.q('.articleCopyText', true);
|
||||
release.description = query.content('.articleCopyText');
|
||||
|
||||
release.actors = actors.map((actor) => actor.trim());
|
||||
release.date = query.date('.articlePostDateText', 'MMMM D, YYYY');
|
||||
release.duration = query.dur('.articlePostDateText a:nth-child(2)');
|
||||
release.duration = query.duration('.articlePostDateText a:nth-child(2)');
|
||||
|
||||
const [cover, ...photos] = query.imgs('img[src*="images"]');
|
||||
release.covers = [cover];
|
||||
@@ -119,7 +120,7 @@ function scrapeSceneLegacy({ query }, site) {
|
||||
|
||||
release.poster = query.poster();
|
||||
|
||||
const trailer = query.trailer();
|
||||
const trailer = query.video('source');
|
||||
if (trailer) release.trailer = { src: trailer };
|
||||
|
||||
return release;
|
||||
@@ -127,10 +128,13 @@ function scrapeSceneLegacy({ query }, site) {
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const url = `${site.parameters.latest}/home.php?o=latest&p=${page}`;
|
||||
const res = await qu.getAll(url, 'body > .columns .column', { cookie: 'consent=yes' });
|
||||
const res = await unprint.get(url, {
|
||||
selectAll: 'body > .columns .column',
|
||||
cookies: { consent: 'yes' },
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeLatest(res.items, site);
|
||||
return scrapeLatest(res.context, site);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
@@ -138,21 +142,22 @@ async function fetchLatest(site, page = 1) {
|
||||
|
||||
async function fetchLatestLegacy(site, page = 1) {
|
||||
const url = `${site.url}/scripts/switch_tour.php?type=brief&page=${page}`;
|
||||
const res = await qu.get(url); // JSON containing html as a property
|
||||
const res = await unprint.get(url); // JSON containing html as a property
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeLatestLegacy(qu.extractAll(res.body.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site);
|
||||
if (res.ok && res.data?.html) {
|
||||
return scrapeLatestLegacy(unprint.initAll(res.data.html, '#articleTable > tbody > tr:nth-child(2) > td > table'), site);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchScene(url, site, baseRelease, parameters) {
|
||||
const session = http.session();
|
||||
const res = await qu.get(url, null, { cookie: 'consent=yes' }, { session });
|
||||
async function fetchScene(url, site, baseRelease, include) {
|
||||
const res = await unprint.get(url, {
|
||||
cookies: { consent: 'yes' },
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeScene(res.item, url, site, parameters, session);
|
||||
return scrapeScene(res.context, site, include, { consent: 'yes', ...res.cookies });
|
||||
}
|
||||
|
||||
return res.status;
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
'use strict';
|
||||
|
||||
const { get, initAll } = require('../utils/qu');
|
||||
const unprint = require('unprint');
|
||||
|
||||
// paths are relative to /visitors/, read raw attributes instead of resolved URLs
|
||||
const raw = { forceGetAttribute: true };
|
||||
|
||||
function scrapeLatest(scenes, dates, site) {
|
||||
return scenes.map(({ qu }, index) => {
|
||||
return scenes.map(({ query }, index) => {
|
||||
const release = {};
|
||||
const path = qu.url('a[href*="videos/"]');
|
||||
const path = query.attribute('a[href*="videos/"]', 'href', raw);
|
||||
|
||||
if (path) {
|
||||
if (/\.wmv$/.test(path)) {
|
||||
@@ -16,30 +19,30 @@ function scrapeLatest(scenes, dates, site) {
|
||||
}
|
||||
|
||||
if (dates && dates[index]) {
|
||||
release.date = dates[index].qu.date(null, 'MM/DD/YYYY');
|
||||
release.date = dates[index].query.date(null, 'MM/DD/YYYY');
|
||||
}
|
||||
|
||||
// release.entryId = release.date ? `${formatDate(release.date, 'YYYY-MM-DD')}-${entryId}` : entryId;
|
||||
release.entryId = path?.match(/videos\/([a-zA-Z0-9]+)(?:_hd)?_trailer/)?.[1]
|
||||
|| qu.img('img[src*="graphics/fft"]')?.match(/fft_(\w+).gif/)?.[1];
|
||||
|| query.attribute('img[src*="graphics/fft"]', 'src', raw)?.match(/fft_(\w+).gif/)?.[1];
|
||||
|
||||
release.description = qu.q('tbody tr:nth-child(3) font', true);
|
||||
release.description = query.content('tbody tr:nth-child(3) font');
|
||||
|
||||
const infoLine = qu.q('font[color="#663366"]', true);
|
||||
const infoLine = query.content('font[color="#663366"]');
|
||||
|
||||
if (infoLine) {
|
||||
release.duration = Number(infoLine.match(/(\d+) min/i)?.[1] || infoLine.match(/video: (\d+)/i)?.[1]) * 60 || null;
|
||||
}
|
||||
|
||||
const poster = qu.img('img[src*="photos/"][width="400"]');
|
||||
const poster = query.attribute('img[src*="photos/"][width="400"]', 'src', raw);
|
||||
release.poster = `${site.url}/visitors/${poster}`;
|
||||
release.photos = qu.imgs('img[src*="photos/"]:not([width="400"])').map((source) => `${site.url}/visitors/${source}`);
|
||||
release.photos = query.attributes('img[src*="photos/"]:not([width="400"])', 'src', raw).map((source) => `${site.url}/visitors/${source}`);
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
function scrapeScene({ qu }, url, site) {
|
||||
function scrapeScene({ query }, url, site) {
|
||||
const release = { url };
|
||||
|
||||
const { pathname } = new URL(url);
|
||||
@@ -52,47 +55,47 @@ function scrapeScene({ qu }, url, site) {
|
||||
return release;
|
||||
}
|
||||
|
||||
const actor = qu.q('font[color="#990033"] strong', true);
|
||||
const actor = query.content('font[color="#990033"] strong');
|
||||
release.actors = [actor];
|
||||
|
||||
const hdTrailer = qu.url('a[href*="hd_trailer.mp4"]');
|
||||
const sdTrailer = qu.url('a[href*="hd_trailer_mobile.mp4"]');
|
||||
const hdTrailer = query.attribute('a[href*="hd_trailer.mp4"]', 'href', raw);
|
||||
const sdTrailer = query.attribute('a[href*="hd_trailer_mobile.mp4"]', 'href', raw);
|
||||
|
||||
release.trailer = [
|
||||
{
|
||||
hdTrailer && {
|
||||
src: `${site.url}/visitors/videos/${hdTrailer}`,
|
||||
quality: 1080,
|
||||
},
|
||||
{
|
||||
sdTrailer && {
|
||||
src: `${site.url}/visitors/videos/${sdTrailer}`,
|
||||
quality: 270,
|
||||
},
|
||||
];
|
||||
].filter(Boolean);
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const url = `https://jesseloadsmonsterfacials.com/visitors/tour_${page.toString().padStart(2, '0')}.html`;
|
||||
const res = await get(url);
|
||||
const res = await unprint.get(url);
|
||||
|
||||
if (!res.ok) {
|
||||
return res.status;
|
||||
}
|
||||
|
||||
const { el } = res.item;
|
||||
const { element } = res.context;
|
||||
|
||||
const scenes = initAll(el, 'table[width="880"]');
|
||||
const dates = initAll(el, 'font[color="#000000"] strong:not(:empty)');
|
||||
const scenes = unprint.initAll(element, 'table[width="880"]');
|
||||
const dates = unprint.initAll(element, 'font[color="#000000"] strong:not(:empty)');
|
||||
|
||||
return scrapeLatest(scenes, dates, site);
|
||||
}
|
||||
|
||||
async function fetchScene(url, site) {
|
||||
const res = await get(url);
|
||||
const res = await unprint.get(url);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeScene(res.item, url, site);
|
||||
return scrapeScene(res.context, url, site);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
const blake2 = require('blake2');
|
||||
|
||||
const qu = require('../utils/qu');
|
||||
const unprint = require('unprint');
|
||||
|
||||
function getHash(identifier) {
|
||||
const hash = blake2.createHash('blake2b', { digestLength: 8 });
|
||||
@@ -45,8 +45,8 @@ async function scrapeLatest(scenes, site) {
|
||||
return scenes.map(({ query }) => {
|
||||
const release = {};
|
||||
|
||||
release.title = query.q('a', 'title');
|
||||
release.url = query.url('a', 'href', { origin: site.url });
|
||||
release.title = query.attribute('a', 'title');
|
||||
release.url = query.url('a', { origin: site.url });
|
||||
release.date = query.date('.nm-date', 'MM/DD/YYYY');
|
||||
|
||||
const slug = new URL(release.url).pathname.split('/')[2];
|
||||
@@ -56,7 +56,7 @@ async function scrapeLatest(scenes, site) {
|
||||
|
||||
[release.poster, ...release.photos] = query.imgs('.bloc-link img');
|
||||
|
||||
release.tags = query.cnts('.dropdown ul a').slice(1);
|
||||
release.tags = query.contents('.dropdown ul a').slice(1);
|
||||
release.duration = query.duration('.dropdown p:first-child');
|
||||
|
||||
return release;
|
||||
@@ -66,22 +66,22 @@ async function scrapeLatest(scenes, site) {
|
||||
async function scrapeScene({ query }, site, url) {
|
||||
const release = { url, site };
|
||||
|
||||
release.title = query.cnt('#movie-header h2');
|
||||
release.date = query.date('#movie-header div span', 'MMMM DD, YYYY', /\w+ \d{1,2}, \d{4}/);
|
||||
release.title = query.content('#movie-header h2');
|
||||
release.date = query.date('#movie-header div span', 'MMMM DD, YYYY', { match: /\w+ \d{1,2}, \d{4}/ });
|
||||
|
||||
release.description = query.cnt('.container .mg-md');
|
||||
release.description = query.content('.container .mg-md');
|
||||
release.duration = query.duration('#video-ribbon .container > div > span:nth-child(3)');
|
||||
|
||||
release.actors = query.cnts('#video-info a').concat(extractMaleModelsFromTags(query.q('.tag-container')));
|
||||
release.tags = query.cnts('.tag-container a');
|
||||
release.actors = query.contents('#video-info a').concat(extractMaleModelsFromTags(query.element('.tag-container')));
|
||||
release.tags = query.contents('.tag-container a');
|
||||
|
||||
const uhd = query.cnt('#video-ribbon .container > div > span:nth-child(2)');
|
||||
const uhd = query.content('#video-ribbon .container > div > span:nth-child(2)');
|
||||
if (/4K/.test(uhd)) release.tags = release.tags.concat('4k');
|
||||
|
||||
release.photos = query.all('.bxslider_pics img').map((el) => el.dataset.original || el.src);
|
||||
release.poster = query.poster();
|
||||
|
||||
const trailer = query.trailer();
|
||||
const trailer = query.video('source');
|
||||
if (trailer) release.trailer = { src: trailer };
|
||||
|
||||
if (release.photos.length > 0) release.channel = await extractChannelFromPhoto(release.photos[0], site);
|
||||
@@ -99,20 +99,20 @@ async function scrapeScene({ query }, site, url) {
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const url = `${site.url}/movies/page-${page}`;
|
||||
const res = await qu.getAll(url, '#content-main [class^="item"]');
|
||||
const res = await unprint.get(url, { selectAll: '#content-main [class^="item"]' });
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeLatest(res.items, site);
|
||||
return scrapeLatest(res.context, site);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchScene(url, channel) {
|
||||
const res = await qu.get(url);
|
||||
const res = await unprint.get(url);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeScene(res.item, channel, url);
|
||||
return scrapeScene(res.context, channel, url);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
|
||||
+46
-39
@@ -1,24 +1,42 @@
|
||||
'use strict';
|
||||
|
||||
const qu = require('../utils/qu');
|
||||
const http = require('../utils/http');
|
||||
const unprint = require('unprint');
|
||||
|
||||
function parseJson(res) {
|
||||
// admin-ajax responds with JSON labeled as text/html
|
||||
if (typeof res.data === 'object') {
|
||||
return res.data;
|
||||
}
|
||||
|
||||
try {
|
||||
return JSON.parse(res.body);
|
||||
} catch (_error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function getTrailer(entryId, channel) {
|
||||
const videoLinksRes = await http.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
|
||||
const videoLinksRes = await unprint.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
|
||||
action: 'vidplay',
|
||||
vid: entryId,
|
||||
type: 'promo',
|
||||
}, {
|
||||
encodeJSON: false,
|
||||
decodeJSON: true,
|
||||
form: true,
|
||||
});
|
||||
|
||||
if (!videoLinksRes.ok || !videoLinksRes.body.size1) {
|
||||
const videoLinks = videoLinksRes.ok && parseJson(videoLinksRes);
|
||||
|
||||
if (!videoLinks?.size1) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const scriptUrl = qu.extract(videoLinksRes.body.size1)?.query.url('script', 'src', { origin: channel.url });
|
||||
const videoScriptRes = await http.get(scriptUrl);
|
||||
const scriptUrl = unprint.init(videoLinks.size1, null, { origin: channel.url })?.query.url('script', { attribute: 'src' });
|
||||
|
||||
if (!scriptUrl) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const videoScriptRes = await unprint.get(scriptUrl, { extract: false });
|
||||
|
||||
if (!videoScriptRes.ok) {
|
||||
return null;
|
||||
@@ -28,13 +46,11 @@ async function getTrailer(entryId, channel) {
|
||||
const videoData = videoDataString && JSON.parse(videoDataString);
|
||||
|
||||
const posterUrlString = videoScriptRes.body.match(/src="(.*poster.*?)"/)?.[1];
|
||||
const posterUrlEl = videoScriptRes.document.createElement('span');
|
||||
|
||||
posterUrlEl.innerHTML = posterUrlString;
|
||||
const poster = posterUrlString && unprint.init(`<span>${posterUrlString}</span>`)?.query.content('span'); // decode HTML entities
|
||||
|
||||
return {
|
||||
trailer: videoData?.url || null,
|
||||
poster: posterUrlEl.textContent || null,
|
||||
poster: poster || null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -48,7 +64,7 @@ function scrapeAll(scenes) {
|
||||
release.title = scene.title;
|
||||
release.description = scene.description;
|
||||
|
||||
release.date = qu.extractDate(scene.info?.post_date, 'YYYY-MM-DD HH:mm:ss') || qu.extractDate(scene.date, 'MMMM D, YYYY'); // eslint-disable-line camelcase
|
||||
release.date = unprint.extractDate(scene.info?.post_date, 'YYYY-MM-DD HH:mm:ss') || unprint.extractDate(scene.date, 'MMMM D, YYYY'); // eslint-disable-line camelcase
|
||||
|
||||
release.poster = scene.poster[0];
|
||||
release.photos = Object.entries(scene).reduce((acc, [key, data]) => {
|
||||
@@ -63,20 +79,20 @@ function scrapeAll(scenes) {
|
||||
});
|
||||
}
|
||||
|
||||
async function scrapeScene({ query }, channel, baseScene) {
|
||||
async function scrapeScene({ query }, { entity, baseRelease }) {
|
||||
const release = {};
|
||||
|
||||
release.title = query.cnt('.title');
|
||||
release.description = query.cnt('.description');
|
||||
release.title = query.content('.title');
|
||||
release.description = query.content('.description');
|
||||
|
||||
release.entryId = query.q('body', 'class').match(/postid-(\d+)/)?.[1];
|
||||
release.entryId = query.attribute('body', 'class')?.match(/postid-(\d+)/)?.[1];
|
||||
|
||||
const { poster, trailer } = await getTrailer(release.entryId, channel) || {};
|
||||
const { poster, trailer } = await getTrailer(release.entryId, entity) || {};
|
||||
|
||||
release.poster = baseScene?.poster || poster;
|
||||
release.photos = poster && baseScene?.poster
|
||||
? [poster, ...baseScene.photos]
|
||||
: baseScene?.photos;
|
||||
release.poster = baseRelease?.poster || poster;
|
||||
release.photos = poster && baseRelease?.poster
|
||||
? [poster, ...baseRelease.photos]
|
||||
: baseRelease?.photos;
|
||||
|
||||
release.trailer = trailer;
|
||||
|
||||
@@ -84,33 +100,24 @@ async function scrapeScene({ query }, channel, baseScene) {
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page) {
|
||||
const res = await http.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
|
||||
const res = await unprint.post('https://pinkyxxx.com/wp-admin/admin-ajax.php', {
|
||||
action: 'vls',
|
||||
limit: 10,
|
||||
offset: (page - 1) * 10,
|
||||
}, {
|
||||
encodeJSON: false,
|
||||
decodeJSON: true,
|
||||
form: true,
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeAll(res.body.listings);
|
||||
const data = res.ok && parseJson(res);
|
||||
|
||||
if (data?.listings) {
|
||||
return scrapeAll(data.listings);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchScene(url, channel, baseScene) {
|
||||
const res = await qu.get(url);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeScene(res.item, channel, baseScene);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
return res.ok ? null : res.status;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchScene,
|
||||
scrapeScene,
|
||||
};
|
||||
|
||||
+10
-11
@@ -3,7 +3,6 @@
|
||||
const unprint = require('unprint');
|
||||
const mime = require('mime');
|
||||
|
||||
const http = require('../utils/http');
|
||||
const slugify = require('../utils/slugify');
|
||||
const { convert } = require('../utils/convert');
|
||||
|
||||
@@ -132,10 +131,10 @@ async function fetchLatest(channel, page, { parameters }) {
|
||||
? `${channel.parent.url}/_next/data/${endpoint}/sites/${parameters.site}.json?sitename=${parameters.site}&order_by=publish_date&sort_by=desc&per_page=30&page=${page}`
|
||||
: `${channel.url}/_next/data/${endpoint}/${parameters.videos || 'videos'}.json?order_by=publish_date&sort_by=desc&per_page=30&page=${page}`;
|
||||
|
||||
const res = await http.get(url);
|
||||
const res = await unprint.get(url);
|
||||
|
||||
if (res.ok && res.body.pageProps?.contents?.data) {
|
||||
return scrapeAll(res.body.pageProps.contents.data, channel, parameters);
|
||||
if (res.ok && res.data.pageProps?.contents?.data) {
|
||||
return scrapeAll(res.data.pageProps.contents.data, channel, parameters);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
@@ -170,12 +169,12 @@ async function fetchScene(url, channel, baseScene, { parameters }) {
|
||||
|
||||
const slug = new URL(url).pathname.split('/').at(-1);
|
||||
const endpoint = await fetchEndpoint(channel);
|
||||
const res = await http.get(`${channel.url}/_next/data/${endpoint}/${parameters.videos || 'videos'}/${slug}.json?slug=${slug}`);
|
||||
const res = await unprint.get(`${channel.url}/_next/data/${endpoint}/${parameters.videos || 'videos'}/${slug}.json?slug=${slug}`);
|
||||
|
||||
if (res.ok && res.body.pageProps?.content) {
|
||||
if (res.ok && res.data.pageProps?.content) {
|
||||
return scrapeScene({
|
||||
...res.body.pageProps.content,
|
||||
serie: res.body.pageProps.playlist?.data[0],
|
||||
...res.data.pageProps.content,
|
||||
serie: res.data.pageProps.playlist?.data[0],
|
||||
}, channel, parameters);
|
||||
}
|
||||
|
||||
@@ -215,10 +214,10 @@ function scrapeProfile(data, channel, scenes, parameters) {
|
||||
|
||||
async function fetchProfile(actor, { channel, parameters }) {
|
||||
const endpoint = await fetchEndpoint(channel);
|
||||
const res = await http.get(`${channel.url}/_next/data/${endpoint}/${parameters.actors || 'models'}/${actor.slug}.json?slug=${actor.slug}`);
|
||||
const res = await unprint.get(`${channel.url}/_next/data/${endpoint}/${parameters.actors || 'models'}/${actor.slug}.json?slug=${actor.slug}`);
|
||||
|
||||
if (res.ok && res.body.pageProps?.model) {
|
||||
return scrapeProfile(res.body.pageProps.model, channel, res.body.pageProps.model_contents, parameters);
|
||||
if (res.ok && res.data.pageProps?.model) {
|
||||
return scrapeProfile(res.data.pageProps.model, channel, res.data.pageProps.model_contents, parameters);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
|
||||
@@ -39,7 +39,6 @@ const karups = require('./karups');
|
||||
const kellymadison = require('./kellymadison');
|
||||
const kink = require('./kink');
|
||||
const mariskax = require('./mariskax');
|
||||
// const analvids = require('./analvids');
|
||||
const pornbox = require('./pornbox');
|
||||
const littlecapricedreams = require('./littlecapricedreams');
|
||||
const loveherfilms = require('./loveherfilms');
|
||||
@@ -62,6 +61,7 @@ const pornworld = require('./pornworld');
|
||||
const privateNetwork = require('./private'); // reserved keyword
|
||||
const radical = require('./radical');
|
||||
const rickysroom = require('./rickysroom');
|
||||
const rsghmedia = require('./rsghmedia');
|
||||
const sexlikereal = require('./sexlikereal');
|
||||
const score = require('./score');
|
||||
const sicflics = require('./sicflics');
|
||||
@@ -73,7 +73,6 @@ const teenmegaworld = require('./teenmegaworld');
|
||||
const testedefudelidade = require('./testedefudelidade');
|
||||
const theflourish = require('./theflourish');
|
||||
const tokyohot = require('./tokyohot');
|
||||
// const topwebmodels = require('./topwebmodels');
|
||||
const traxxx = require('./traxxx');
|
||||
const virtualtaboo = require('./virtualtaboo');
|
||||
const vixen = require('./vixen');
|
||||
@@ -102,6 +101,7 @@ module.exports = {
|
||||
acam,
|
||||
analvids: pornbox,
|
||||
pornbox,
|
||||
cosplayground: aziani,
|
||||
kellymadison,
|
||||
kink,
|
||||
'8kmembers': kellymadison,
|
||||
@@ -143,6 +143,7 @@ module.exports = {
|
||||
interracialpass: hush,
|
||||
inthecrack,
|
||||
jerkaoke: modelmedia,
|
||||
jenniferwhite: adultempire,
|
||||
jesseloadsmonsterfacials,
|
||||
julesjordan,
|
||||
karups,
|
||||
@@ -170,6 +171,7 @@ module.exports = {
|
||||
private: privateNetwork,
|
||||
radical,
|
||||
rickysroom,
|
||||
rsghmedia,
|
||||
score,
|
||||
sexlikereal,
|
||||
sicflics,
|
||||
|
||||
+46
-44
@@ -1,31 +1,36 @@
|
||||
'use strict';
|
||||
|
||||
const qu = require('../utils/q');
|
||||
const unprint = require('unprint');
|
||||
|
||||
const { lbsToKg, feetInchesToCm } = require('../utils/convert');
|
||||
|
||||
function scrapeScene(data, channel) {
|
||||
const release = {};
|
||||
|
||||
release.entryId = data.id;
|
||||
release.url = qu.prefixUrl(`/videos/${data.slug}`, channel.url);
|
||||
release.url = unprint.prefixUrl(`/videos/${data.slug}`, channel.url);
|
||||
|
||||
release.title = data.title;
|
||||
release.description = data.description;
|
||||
release.date = qu.parseDate(data.publish_date, 'YYYY/MM/DD HH:mm:ss');
|
||||
release.duration = qu.durationToSeconds(data.videos_duration);
|
||||
release.date = unprint.extractDate(data.publish_date, 'YYYY/MM/DD HH:mm:ss');
|
||||
release.duration = unprint.extractDuration(data.videos_duration);
|
||||
|
||||
release.actors = data?.models_thumbs.map((model) => ({
|
||||
name: model.name,
|
||||
url: qu.prefixUrl(`/models/${model.slug}`, channel.url),
|
||||
url: unprint.prefixUrl(`/models/${model.slug}`, channel.url),
|
||||
avatar: model.thumb,
|
||||
})) || data.models;
|
||||
|
||||
release.tags = data.tags;
|
||||
|
||||
release.poster = [data.trailer_screencap].concat(data.extra_thumbnails);
|
||||
release.photos = data.previews?.full
|
||||
.map((url) => [url, url.replace('full/', 'thumbs/')]) // photos
|
||||
.concat(data.thumbs); // screenshots
|
||||
|
||||
// inconsistent data structure. if there is an album, it's previews: { full: [...] }, if it's empty, it's previews: []
|
||||
if (Array.isArray(data.previews?.full)) {
|
||||
release.photos = data.previews?.full
|
||||
.map((url) => [url, url.replace('full/', 'thumbs/')]) // photos
|
||||
.concat(data.thumbs); // screenshots
|
||||
}
|
||||
|
||||
release.trailer = data.trailer_url;
|
||||
release.teaser = data.special_thumbnails;
|
||||
@@ -36,39 +41,14 @@ function scrapeScene(data, channel) {
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeProfile(data, scenes, entity) {
|
||||
const profile = {};
|
||||
|
||||
profile.entryId = data.id;
|
||||
profile.url = qu.prefixUrl(`/models/${data.slug}`, entity.url);
|
||||
|
||||
profile.description = data.Bio || data.bio;
|
||||
profile.birthPlace = data.Born || data.born;
|
||||
profile.dateOfBirth = qu.parseDate(data.Birthdate || data.birthdate, 'YYYY-MM-DD');
|
||||
|
||||
profile.measurements = data.Measurements || data.Measurements;
|
||||
profile.height = feetInchesToCm(data.Height || data.height);
|
||||
profile.weight = lbsToKg(data.Weight || data.weight);
|
||||
|
||||
profile.eyes = data.Eyes || data.eyes;
|
||||
profile.hairColor = data.Hair || data.hair;
|
||||
|
||||
profile.avatar = data.thumb;
|
||||
|
||||
profile.scenes = scenes?.map((scene) => scrapeScene(scene, entity));
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page = 1) {
|
||||
const url = `${channel.url}/videos?order_by=publish_date&per_page=100&page=${page}`; // unsure if page works, not enough videos as of 2022-05-29
|
||||
const res = await qu.get(url);
|
||||
const res = await unprint.get(url);
|
||||
|
||||
if (res.ok) {
|
||||
const dataString = res.item.query.html('#__NEXT_DATA__');
|
||||
const data = dataString && JSON.parse(dataString);
|
||||
const data = res.context.query.json('#__NEXT_DATA__');
|
||||
|
||||
if (data.props?.pageProps?.contents?.data) {
|
||||
if (data?.props?.pageProps?.contents?.data) {
|
||||
const scenes = data.props.pageProps.contents.data.map((scene) => scrapeScene(scene, channel));
|
||||
|
||||
return scenes;
|
||||
@@ -86,13 +66,12 @@ async function fetchScene(url, channel, baseRelease) {
|
||||
return baseRelease;
|
||||
}
|
||||
|
||||
const res = await qu.get(url);
|
||||
const res = await unprint.get(url);
|
||||
|
||||
if (res.ok) {
|
||||
const dataString = res.item.query.html('#__NEXT_DATA__');
|
||||
const data = dataString && JSON.parse(dataString);
|
||||
const data = res.context.query.json('#__NEXT_DATA__');
|
||||
|
||||
if (data.props?.pageProps?.content) {
|
||||
if (data?.props?.pageProps?.content) {
|
||||
return scrapeScene(data.props.pageProps.content, channel);
|
||||
}
|
||||
|
||||
@@ -102,15 +81,38 @@ async function fetchScene(url, channel, baseRelease) {
|
||||
return res.status;
|
||||
}
|
||||
|
||||
function scrapeProfile(data, scenes, entity) {
|
||||
const profile = {};
|
||||
|
||||
profile.entryId = data.id;
|
||||
profile.url = unprint.prefixUrl(`/models/${data.slug}`, entity.url);
|
||||
|
||||
profile.description = data.Bio || data.bio;
|
||||
profile.birthPlace = data.Born || data.born;
|
||||
profile.dateOfBirth = unprint.extractDate(data.Birthdate || data.birthdate, 'YYYY-MM-DD');
|
||||
|
||||
profile.measurements = data.Measurements || data.Measurements;
|
||||
profile.height = feetInchesToCm(data.Height || data.height);
|
||||
profile.weight = lbsToKg(data.Weight || data.weight);
|
||||
|
||||
profile.eyes = data.Eyes || data.eyes;
|
||||
profile.hairColor = data.Hair || data.hair;
|
||||
|
||||
profile.avatar = data.thumb;
|
||||
|
||||
profile.scenes = scenes?.map((scene) => scrapeScene(scene, entity));
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchProfile({ slug }, entity) {
|
||||
const url = `${entity.url}/models/${slug}`;
|
||||
const res = await qu.get(url);
|
||||
const res = await unprint.get(url);
|
||||
|
||||
if (res.ok) {
|
||||
const dataString = res.item.query.html('#__NEXT_DATA__');
|
||||
const data = dataString && JSON.parse(dataString);
|
||||
const data = res.context.query.json('#__NEXT_DATA__');
|
||||
|
||||
if (data.props?.pageProps?.model) {
|
||||
if (data?.props?.pageProps?.model) {
|
||||
return scrapeProfile(data.props.pageProps.model, data.props.pageProps.model_contents, entity);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
'use strict';
|
||||
|
||||
const unprint = require('unprint');
|
||||
const escapeRegexp = require('escape-string-regexp');
|
||||
|
||||
const slugify = require('../utils/slugify');
|
||||
const capitalize = require('../utils/capitalize');
|
||||
|
||||
function getDate(query, release) {
|
||||
const date = query.date('.published', 'MMMM D, YYYY') || query.date('meta[itemprop="uploadDate"]', 'YYYY-MM-DD', { attribute: 'content' });
|
||||
|
||||
if (date) {
|
||||
return {
|
||||
date,
|
||||
datePrecision: 'day',
|
||||
};
|
||||
}
|
||||
|
||||
const poster = Array.isArray(release.poster)
|
||||
? release.poster[0]
|
||||
: release.poster;
|
||||
|
||||
if (poster) {
|
||||
const [, year, month] = new URL(poster).pathname.match(/\/uploads\/(\d{4})\/(\d{2})/) || [];
|
||||
|
||||
if (year && month) {
|
||||
return {
|
||||
date: new Date(Date.UTC(year, month - 1, 1)),
|
||||
datePrecision: 'month',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
function scrapeAll(scenes) {
|
||||
return scenes.map(({ query, element }) => {
|
||||
const release = {};
|
||||
|
||||
release.url = query.url('.entry-title a, a.entry-featured-image-url');
|
||||
release.entryId = new URL(release.url).pathname.match(/\/([\w-]+)/)[1];
|
||||
|
||||
const postId = unprint.query.number(element, null, { attribute: 'id' });
|
||||
|
||||
if (postId) {
|
||||
release.attributes = { postId };
|
||||
}
|
||||
|
||||
release.title = query.content('.entry-title a');
|
||||
// description is cut off
|
||||
|
||||
release.tags = query.contents('.post-meta a[href*="/category"], .post-meta a[href*="/tag"]'); // tag only seen on scene page, but just in case
|
||||
release.poster = Array.from(new Set(query.sourceSet('.entry-featured-image-url img')));
|
||||
|
||||
const { date, datePrecision } = getDate(query, release);
|
||||
|
||||
release.date = date;
|
||||
release.datePrecision = datePrecision;
|
||||
|
||||
return release;
|
||||
}).toSorted((sceneA, sceneB) => {
|
||||
// HTML is laid out in columns that does not align with chronology
|
||||
if (sceneA.attributes?.postId && sceneB.attributes?.postId) {
|
||||
return sceneB.attributes.postId - sceneA.attributes.postId;
|
||||
}
|
||||
|
||||
if (sceneA.date && sceneB.date) {
|
||||
return sceneB.date - sceneA.date;
|
||||
}
|
||||
|
||||
return 0;
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page = 1) {
|
||||
const url = `${channel.origin}/page/${page}`;
|
||||
const res = await unprint.get(url, { selectAll: '.et_pb_blog_grid .et_pb_post' });
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeAll(res.context, channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
function scrapeScene({ query }, { url }) {
|
||||
const release = {};
|
||||
|
||||
release.entryId = new URL(url).pathname.match(/\/([\w-]+)/)[1];
|
||||
|
||||
const postId = query.number('article[id*="post"]', { attribute: 'id' });
|
||||
|
||||
if (postId) {
|
||||
release.attributes = { postId };
|
||||
}
|
||||
|
||||
release.title = query.content('.entry-title');
|
||||
release.description = query.content('.entry-content p');
|
||||
|
||||
release.actors = query.all('.st-post-tags a[href*="/tag"]').map((actorEl) => ({
|
||||
name: unprint.query.content(actorEl),
|
||||
url: unprint.query.url(actorEl, null),
|
||||
}));
|
||||
|
||||
release.poster = query.img('meta[property="og:image"], meta[property="twitter:image"]', { attribute: 'content' })
|
||||
|| query.sourceSet('.et_post_meta_wrapper img');
|
||||
|
||||
release.trailer = query.video('.entry-content source');
|
||||
|
||||
const { date, datePrecision } = getDate(query, release);
|
||||
|
||||
release.date = date;
|
||||
release.datePrecision = datePrecision;
|
||||
|
||||
const tags = query.all('.post-meta a[href*="/category"], .post-meta a[href*="/tag"]');
|
||||
const starring = query.content('//div[contains(@class, "entry-content")]/p[contains(text(), "Starring")]')
|
||||
|| query.content('//div[contains(@class, "entry-content")]/p[contains(text(), "***")]')
|
||||
|| release.title?.split(/\s[-–]\s/).at(-1);
|
||||
|
||||
if (tags.length > 0) {
|
||||
release.tags = tags.map((tagEl) => unprint.query.content(tagEl));
|
||||
} else {
|
||||
release.tags = query.content('article[id*="post"]', { attribute: 'class' })
|
||||
?.match(/(category|tag)-[\w-]+/g)
|
||||
?.map((tag) => slugify(tag.replace(/(category|tag)-/, ''), ' '))
|
||||
|| [];
|
||||
}
|
||||
|
||||
if (release.actors.length === 0 && release.tags.length > 0 && starring) {
|
||||
// actors are always added as tags, sometimes they're clearly divided, sometimes they're mixed in with other tags
|
||||
// the tail end of the description or title contains a *starring* section against which the tags can be matched
|
||||
release.actors = release.tags
|
||||
.map((tag) => {
|
||||
if (tag === 'alex adams') {
|
||||
// household male performer that is not usually mentioned in the starring section
|
||||
return capitalize(tag);
|
||||
}
|
||||
|
||||
// use description text instead of tag name for capitalization
|
||||
const match = starring.match(new RegExp(`\\b${escapeRegexp(tag)}\\b`, 'i'))?.[0];
|
||||
|
||||
if (match) {
|
||||
return match;
|
||||
}
|
||||
|
||||
return null;
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
scrapeScene,
|
||||
};
|
||||
+21
-25
@@ -2,7 +2,6 @@
|
||||
|
||||
const unprint = require('unprint');
|
||||
|
||||
const http = require('../utils/http');
|
||||
const slugify = require('../utils/slugify');
|
||||
const { convert } = require('../utils/convert');
|
||||
|
||||
@@ -34,7 +33,8 @@ async function fetchTrailerUrl(videoId, entity) {
|
||||
const token = res.data.token;
|
||||
|
||||
if (token) {
|
||||
return `https://customer-bk3o9te23pydwwcb.cloudflarestream.com/${token}/manifest/video.mpd?parentOrigin=${entity.origin}`;
|
||||
// ffmpeg fails to read the DASH manifest over a single connection, HLS works
|
||||
return `https://customer-bk3o9te23pydwwcb.cloudflarestream.com/${token}/manifest/video.m3u8?parentOrigin=${entity.origin}`;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,17 +47,18 @@ async function scrapeScene(scene, channel, _parameters, includeTrailers) {
|
||||
// release.entryId = scene.id; // legacy
|
||||
release.entryId = scene.itemId;
|
||||
|
||||
release.url = `${channel.type === 'network' || channel.parameters?.layout === 'organic' ? channel.url : channel.parent.url}/movies/${release.entryId}`;
|
||||
release.url = `${channel.origin}/movies/${scene.id || release.entryId}`; // numeric item ID URL 404s, id is the slug
|
||||
|
||||
release.title = scene.title;
|
||||
release.description = scene.description;
|
||||
release.date = unprint.extractDate(scene.publishedDate, 'YYYY-MM-DD');
|
||||
release.duration = scene.d;
|
||||
|
||||
release.actors = scene.models?.map((model) => ({
|
||||
name: model.modelName || model.name || model.title,
|
||||
avatar: model.img,
|
||||
gender: model.gender,
|
||||
url: `${channel.url}/models/${model.modelId || model.id}`,
|
||||
url: `${channel.origin}/models/${model.modelId || model.id}`,
|
||||
}));
|
||||
|
||||
if (scene.img) {
|
||||
@@ -93,16 +94,16 @@ function scrapeAll(scenes, channel, parameters) {
|
||||
|
||||
async function fetchLatest(channel, page = 1, { parameters }) {
|
||||
// url: 'https://www.pervz.com/series/pervz-features',
|
||||
const seriesId = parameters.id || new URL(channel.url).pathname.match(/\/series\/([a-z-]+)/)?.[1];
|
||||
const seriesId = parameters.id || new URL(channel.url).pathname.match(/\/series\/([\w-]+)/)?.[1];
|
||||
|
||||
const url = seriesId
|
||||
? `https://tours-store.psmcdn.net/${parameters.endpoint}/_search?q=(site.seo.seoSlug:"${seriesId}" AND type:video)&sort=publishedDate:desc&size=30&from=${(page - 1) * 30}`
|
||||
: `https://tours-store.psmcdn.net/${parameters.endpoint}/_search?sort=publishedDate:desc&q=(type:video AND isXSeries:false)&size=30&from=${(page - 1) * 30}`;
|
||||
|
||||
const res = await http.get(url);
|
||||
const res = await unprint.get(url);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeAll(res.body.hits.hits.map(({ _source: scene }) => scene), channel, parameters);
|
||||
if (res.ok && res.data?.hits) {
|
||||
return scrapeAll(res.data.hits.hits.map(({ _source: scene }) => scene), channel, parameters);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
@@ -114,27 +115,22 @@ async function fetchScene(url, channel, baseScene, { parameters, includeTrailers
|
||||
return baseScene;
|
||||
}
|
||||
|
||||
const sceneSlug = new URL(url).pathname.match(/\/([\w-]+$)/)[1];
|
||||
const sceneSlug = new URL(url).pathname.match(/\/([\w-]+)\/?$/)?.[1];
|
||||
|
||||
const res = await unprint.get(url, {
|
||||
parser: {
|
||||
runScripts: 'dangerously',
|
||||
},
|
||||
});
|
||||
if (!sceneSlug) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (res.ok) {
|
||||
const videos = res.context.window.__INITIAL_STATE__?.content?.videosContent;
|
||||
// scene page no longer embeds state, look up the scene in the elastic API by its slug instead
|
||||
// legacy URLs use the numeric item ID, which 404s on the site but can still be looked up
|
||||
const isItemId = /^\d+$/.test(sceneSlug);
|
||||
const res = await unprint.get(`https://tours-store.psmcdn.net/${parameters.endpoint}/_search?q=(${isItemId ? `itemId:${sceneSlug}` : `id:"${sceneSlug}"`} AND type:video)&size=1`);
|
||||
|
||||
res.context.window.fetch = () => {}; // suppress fetch missing error
|
||||
if (res.ok && res.data?.hits) {
|
||||
const scene = res.data.hits.hits.find(({ _source: hit }) => (isItemId ? String(hit.itemId) : hit.id) === sceneSlug)?._source;
|
||||
|
||||
if (!videos) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const video = videos?.[sceneSlug] || Object.values(videos)[0];
|
||||
|
||||
if (video) {
|
||||
return scrapeScene(video, channel, parameters, includeTrailers);
|
||||
if (scene) {
|
||||
return scrapeScene(scene, channel, parameters, includeTrailers);
|
||||
}
|
||||
|
||||
return null;
|
||||
|
||||
@@ -31,7 +31,7 @@ function scrapeAll(scenes) {
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page = 1) {
|
||||
const url = `${channel.url}/${page}`;
|
||||
const url = `${channel.origin}/${page}`;
|
||||
const res = await unprint.get(url, { selectAll: '.scene' });
|
||||
|
||||
if (res.ok) {
|
||||
|
||||
@@ -1,144 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const mime = require('mime');
|
||||
|
||||
const qu = require('../utils/qu');
|
||||
const http = require('../utils/http');
|
||||
const slugify = require('../utils/slugify');
|
||||
const { convert } = require('../utils/convert');
|
||||
|
||||
function scrapeSceneX(scene) {
|
||||
const release = {};
|
||||
|
||||
release.entryId = scene.id;
|
||||
release.url = `https://tour.topwebmodels.com/scenes/${scene.id}/${slugify(scene.title, '-', { removePunctuation: true })}`;
|
||||
|
||||
release.title = scene.title;
|
||||
release.description = scene.description;
|
||||
|
||||
release.duration = qu.durationToSeconds(scene.videos_duration);
|
||||
release.date = new Date(scene.release_date);
|
||||
|
||||
release.actors = scene.models
|
||||
.map((actor) => (/&/.test(actor.name)
|
||||
? actor.name.split(/\s*&\s*/)
|
||||
: {
|
||||
name: actor.name,
|
||||
gender: actor.gender || null,
|
||||
avatar: actor.thumb,
|
||||
url: `https://tour.topwebmodels.com/models/${actor.id}/${slugify(actor.name)}`,
|
||||
}))
|
||||
.flat();
|
||||
|
||||
release.stars = scene.rating;
|
||||
release.tags = scene.tags.map((tag) => tag.name);
|
||||
|
||||
if (mime.getType(scene.thumb) === 'image/gif') {
|
||||
release.teaser = scene.thumb;
|
||||
} else {
|
||||
release.poster = scene.thumb;
|
||||
}
|
||||
|
||||
release.channel = slugify(scene.sites[0]?.name, '');
|
||||
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeAll(scenes) {
|
||||
return scenes.map(scrapeSceneX);
|
||||
}
|
||||
|
||||
async function scrapeProfile(actor, options) {
|
||||
const profile = {};
|
||||
|
||||
profile.dateOfBirth = /1969-12-31/.test(actor.attributes.birthdate.value) ? null : qu.extractDate(actor.attributes.birthdate.value, 'YYYY-MM-DD'); // ignore epoch
|
||||
profile.age = actor.attributes.age.value === 51 ? null : actor.attributes.age.value; // ignore epoch
|
||||
profile.gender = actor.attributes.gender || null;
|
||||
|
||||
profile.height = convert(actor.attributes.height.value, 'cm');
|
||||
profile.weight = convert(actor.attributes.weight.value, 'lb', 'kg');
|
||||
|
||||
const [bust, cup, waist, hip] = actor.attributes.measurements.value?.match(/(\d+)(\w+)-(\d+)-(\d+)/)?.slice(1) || [];
|
||||
|
||||
profile.bust = Number(bust);
|
||||
profile.cup = cup;
|
||||
profile.waist = Number(waist);
|
||||
profile.hip = Number(hip);
|
||||
|
||||
profile.ethnicity = actor.attributes.ethnicity.value;
|
||||
profile.birthPlace = actor.attributes.born.value;
|
||||
|
||||
profile.eyes = actor.attributes.eyes.value;
|
||||
profile.hairColor = actor.attributes.hair.value.split('/')[0];
|
||||
|
||||
profile.url = `https://tour.topwebmodels.com/models/${actor.id}/${slugify(actor.name, '-', { removePunctuation: true })}`;
|
||||
profile.avatar = actor.thumb;
|
||||
|
||||
if (options.includeActorScenes) {
|
||||
const res = await http.get(profile.url, { extract: { runScripts: 'dangerously' } });
|
||||
|
||||
if (res.ok) {
|
||||
profile.scenes = scrapeAll(res.window.__DATA__?.data?.videos?.items);
|
||||
}
|
||||
}
|
||||
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page) {
|
||||
const res = await http.get(`https://tour.topwebmodels.com/api/sites/${channel.parameters?.slug || channel.slug}?page=${page}`, {
|
||||
headers: {
|
||||
Referer: 'https://tour.topwebmodels.com',
|
||||
'api-key': channel.parameters?.apiKey || channel.parent?.parameters?.apiKey,
|
||||
'x-Requested-With': 'XMLHttpRequest',
|
||||
},
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeAll(res.body.videos.items);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchScene(url) {
|
||||
const res = await http.get(url, { extract: { runScripts: 'dangerously' } });
|
||||
|
||||
if (res.ok) {
|
||||
return {
|
||||
...scrapeSceneX(res.window.__DATA__.data.video),
|
||||
...(/\.gif/.test(res.window.__DATA__.data.video.thumb) && { teaser: res.window.__DATA__.data.video.thumb }),
|
||||
poster: res.window.__DATA__.data.file_poster,
|
||||
};
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchProfile(baseActor, entity, options) {
|
||||
const searchRes = await http.get(`https://tour.topwebmodels.com/api/search-preview/${baseActor.name}`, {
|
||||
headers: {
|
||||
Referer: 'https://tour.topwebmodels.com',
|
||||
'api-key': entity.parameters?.apiKey,
|
||||
'x-Requested-With': 'XMLHttpRequest',
|
||||
},
|
||||
});
|
||||
|
||||
if (!searchRes.ok) {
|
||||
return searchRes.status;
|
||||
}
|
||||
|
||||
const actor = searchRes.body.models.items.find((model) => slugify(model.name) === slugify(baseActor.name));
|
||||
|
||||
if (actor) {
|
||||
return scrapeProfile(actor, options);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchScene,
|
||||
fetchProfile,
|
||||
};
|
||||
@@ -74,7 +74,9 @@ function scrapeAll(scenes, channel) {
|
||||
|
||||
async function fetchLatest(site, page = 1) {
|
||||
const url = `${site.url}/videos?page=${page}`;
|
||||
const res = await unprint.get(url);
|
||||
const res = await unprint.get(url, {
|
||||
interface: 'request',
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
const data = res.context.query.json('#__NEXT_DATA__');
|
||||
|
||||
@@ -12,7 +12,6 @@ const slugify = require('./utils/slugify');
|
||||
const bulkInsert = require('./utils/bulk-insert');
|
||||
const chunk = require('./utils/chunk');
|
||||
const redis = require('./redis');
|
||||
const { formatDate } = require('./utils/qu');
|
||||
const { associateActors, associateDirectors, scrapeActors, toBaseActors } = require('./actors');
|
||||
const { associateReleaseTags } = require('./tags');
|
||||
const { curateEntity } = require('./entities');
|
||||
@@ -32,7 +31,7 @@ const commonContext = {
|
||||
async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
|
||||
const slugBase = release.title
|
||||
|| (release.actors?.length && `${release.entity.slug} ${release.actors.map((actor) => actor.name).join(' ')}`)
|
||||
|| (release.date && `${release.entity.slug} ${formatDate(release.date, 'YYYY MM DD')}`)
|
||||
|| (release.date && `${release.entity.slug} ${unprint.formatDate(release.date, 'YYYY MM DD')}`)
|
||||
|| null;
|
||||
|
||||
const slug = slugify(slugBase, '-', {
|
||||
|
||||
+5
-4
@@ -79,13 +79,14 @@ function withRelations(queryBuilder, withMedia) {
|
||||
}
|
||||
|
||||
async function matchTags(rawTags) {
|
||||
const tags = rawTags
|
||||
.map((tag) => tag?.trim().match(/[a-z0-9()-]+/ig)?.join(' ').toLowerCase())
|
||||
.filter(Boolean);
|
||||
const tags = Array.from(new Set(rawTags
|
||||
.map((tag) => (typeof tag === 'string' ? tag : null)?.trim().match(/[a-z0-9()-]+/ig)?.join(' ').toLowerCase())
|
||||
.filter(Boolean)));
|
||||
|
||||
// array binding, large batches can easily exceed the bindings limit otherwise
|
||||
const tagEntries = await knex('tags')
|
||||
.select('tags.id', 'tags.name', 'tags.alias_for', 'tags.implied_tag_ids')
|
||||
.whereIn(knex.raw('lower(tags.name)'), tags);
|
||||
.whereRaw('lower(tags.name) = ANY(:tags)', { tags });
|
||||
|
||||
const tagIdsBySlug = tagEntries
|
||||
.reduce((acc, tag) => ({
|
||||
|
||||
@@ -1,76 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs').promises;
|
||||
|
||||
const Promise = require('bluebird');
|
||||
|
||||
const qu = require('../utils/qu');
|
||||
|
||||
const qualities = {
|
||||
sm: 360,
|
||||
med: 480,
|
||||
big: 720,
|
||||
};
|
||||
|
||||
/*
|
||||
async function scrape() {
|
||||
const urlsByPage = await Promise.map(Array.from({ length: 140 }), async (value, index) => {
|
||||
const res = await qu.get(`https://www.dogfartnetwork.com/tour/scenes/?p=${index + 1}`);
|
||||
|
||||
if (res.ok) {
|
||||
return res.item.query.urls('.recent-updates > a', 'href', { origin: 'https://www.dogfartnetwork.com' });
|
||||
}
|
||||
|
||||
return [];
|
||||
}, { concurrency: 1 });
|
||||
|
||||
const urls = urlsByPage.flat();
|
||||
|
||||
await fs.writeFile('./dogfart-links', urls.join('\n'));
|
||||
|
||||
console.log(`Saved ${urls.length} URLs to file`);
|
||||
}
|
||||
|
||||
async function compare() {
|
||||
const newLinksFile = await fs.readFile('./dogfart-links', 'utf8');
|
||||
const oldLinksFile = await fs.readFile('./dogfart-got', 'utf8');
|
||||
|
||||
const newLinks = newLinksFile.split('\n').filter(Boolean);
|
||||
const oldLinks = new Set(oldLinksFile.split('\n').filter(Boolean));
|
||||
|
||||
const getLinks = newLinks.filter((link) => !oldLinks.has(link)).map((link) => `https://dogfartnetwork.com/tour/sites${link}`);
|
||||
|
||||
await fs.writeFile('./tmp/dogfart-new', getLinks.join('\n'));
|
||||
|
||||
console.log(getLinks);
|
||||
}
|
||||
*/
|
||||
|
||||
async function scrapeMembers() {
|
||||
const titlesByPage = await Promise.map(Array.from({ length: 20 }), async (value, index) => {
|
||||
const res = await qu.get(`https://sbt1nb9b98m.dogfartnetwork.com/members/gloryholesandhandjobs/index.php?page=${index + 1}`);
|
||||
|
||||
if (res.ok) {
|
||||
return qu.initAll(res.item.query.all('.scene-container')).map(({ query }) => ({
|
||||
url: `https://www.dogfartnetwork.com/tour/sites/GloryholesAndHandjobs/${query.img('.video-container img').match(/\/(\w+).jpg/)[1]}/`,
|
||||
actors: query.contents('a[href*="model.php"]'),
|
||||
trailer: query.urls('.trailer-link a').map((url) => ({
|
||||
src: url,
|
||||
quality: qualities[url.match(/_([a-z]+).mp4/)[1]],
|
||||
})),
|
||||
}));
|
||||
}
|
||||
|
||||
return [];
|
||||
}, { concurrency: 1 });
|
||||
|
||||
const urls = titlesByPage.flat().map((data) => JSON.stringify(data));
|
||||
|
||||
console.log(urls);
|
||||
|
||||
await fs.writeFile('./tmp/dogfart-gloryholesandhandjobs', Array.from(new Set(urls)).join('\n'));
|
||||
|
||||
console.log(`Saved ${urls.length} URLs to file`);
|
||||
}
|
||||
|
||||
scrapeMembers();
|
||||
@@ -1,110 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const config = require('config');
|
||||
const fs = require('fs');
|
||||
const fsPromises = require('fs').promises;
|
||||
const Promise = require('bluebird');
|
||||
const blake2 = require('blake2');
|
||||
const sharp = require('sharp');
|
||||
const { nanoid } = require('nanoid');
|
||||
const { PassThrough } = require('stream');
|
||||
|
||||
const http = require('./http');
|
||||
|
||||
function getMemoryUsage() {
|
||||
return process.memoryUsage().rss / (10 ** 6);
|
||||
}
|
||||
|
||||
let peakMemoryUsage = getMemoryUsage();
|
||||
|
||||
async function fetchSource(link) {
|
||||
const id = nanoid();
|
||||
|
||||
const hasher = new blake2.Hash('blake2b');
|
||||
hasher.setEncoding('hex');
|
||||
|
||||
const tempFilePath = `/home/niels/Pictures/thumbs/temp/${id}.jpeg`;
|
||||
const tempFileStream = fs.createWriteStream(tempFilePath);
|
||||
const hashStream = new PassThrough();
|
||||
|
||||
hashStream.on('data', (chunk) => hasher.write(chunk));
|
||||
|
||||
try {
|
||||
const res = await http.get(link, null, {
|
||||
stream: true,
|
||||
transforms: [hashStream],
|
||||
destination: tempFileStream,
|
||||
timeout: 5000,
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(res.status);
|
||||
}
|
||||
|
||||
hasher.end();
|
||||
const hash = hasher.read();
|
||||
|
||||
const memoryUsage = getMemoryUsage();
|
||||
peakMemoryUsage = Math.max(memoryUsage, peakMemoryUsage);
|
||||
|
||||
console.log(`Stored ${tempFilePath}, memory usage: ${memoryUsage.toFixed(2)} MB`);
|
||||
|
||||
return {
|
||||
id,
|
||||
path: tempFilePath,
|
||||
hash,
|
||||
};
|
||||
} catch (error) {
|
||||
await fsPromises.unlink(tempFilePath);
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function init() {
|
||||
const linksFile = await fsPromises.readFile('/home/niels/Pictures/photos', 'utf8');
|
||||
const links = linksFile.split('\n').filter(Boolean);
|
||||
|
||||
await fsPromises.mkdir('/home/niels/Pictures/thumbs/temp', { recursive: true });
|
||||
|
||||
console.time('thumbs');
|
||||
|
||||
const files = await Promise.map(links, async (link) => {
|
||||
try {
|
||||
return await fetchSource(link);
|
||||
} catch (error) {
|
||||
console.log(`Failed to fetch ${link}: ${error.message}`);
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.map(files.filter(Boolean), async (file) => {
|
||||
const image = sharp(file.path).jpeg();
|
||||
|
||||
const [{ width, height }, { size }] = await Promise.all([
|
||||
image.metadata(),
|
||||
fsPromises.stat(file.path),
|
||||
]);
|
||||
|
||||
await Promise.all([
|
||||
image
|
||||
.toFile(`/home/niels/Pictures/thumbs/${file.hash}.jpeg`),
|
||||
image
|
||||
.resize({
|
||||
height: config.media.thumbnailSize,
|
||||
withoutEnlargement: true,
|
||||
})
|
||||
.toFile(`/home/niels/Pictures/thumbs/${file.hash}_thumb.jpeg`),
|
||||
]);
|
||||
|
||||
const memoryUsage = getMemoryUsage();
|
||||
peakMemoryUsage = Math.max(memoryUsage, peakMemoryUsage);
|
||||
|
||||
console.log(`Resized ${file.id} (${width}, ${height}, ${size}), memory usage: ${memoryUsage.toFixed(2)} MB`);
|
||||
}, { concurrency: 10 });
|
||||
|
||||
console.log(`Peak memory usage: ${peakMemoryUsage.toFixed(2)} MB`);
|
||||
console.timeEnd('thumbs');
|
||||
}
|
||||
|
||||
init();
|
||||
@@ -1,5 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const qu = require('./qu');
|
||||
|
||||
module.exports = qu;
|
||||
-682
@@ -1,682 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const { JSDOM } = require('jsdom');
|
||||
const moment = require('moment');
|
||||
const http = require('./http');
|
||||
const virtualConsole = require('./virtual-console')(__filename);
|
||||
|
||||
const { window: globalWindow } = new JSDOM('', { virtualConsole });
|
||||
|
||||
function trim(str) {
|
||||
if (typeof str !== 'string') {
|
||||
return str;
|
||||
}
|
||||
|
||||
return str.trim().replace(/\s+/g, ' ');
|
||||
}
|
||||
|
||||
function extractDate(dateString, format, match) {
|
||||
if (!dateString) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (match) {
|
||||
const dateStamp = trim(dateString).match(match);
|
||||
|
||||
if (dateStamp) {
|
||||
const dateValue = moment.utc(dateStamp[0], format);
|
||||
|
||||
return dateValue.isValid() ? dateValue.toDate() : null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const dateValue = moment.utc(trim(dateString), format);
|
||||
|
||||
return dateValue.isValid() ? dateValue.toDate() : null;
|
||||
}
|
||||
|
||||
function formatDate(dateValue, format, inputFormat) {
|
||||
if (inputFormat) {
|
||||
return moment(dateValue, inputFormat).format(format);
|
||||
}
|
||||
|
||||
return moment(dateValue).format(format);
|
||||
}
|
||||
|
||||
function durationToSeconds(durationString, match) {
|
||||
const durationMatch = durationString.match(match || /(\d+:)?\d+:\d+/);
|
||||
|
||||
if (durationMatch) {
|
||||
const segments = ['00'].concat(durationMatch[0].split(/[:hm]/)).slice(-3);
|
||||
|
||||
return moment.duration(segments.join(':')).asSeconds();
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function prefixUrl(urlValue, origin, protocol = 'https') {
|
||||
if (!urlValue) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (/^http/.test(urlValue)) {
|
||||
return urlValue;
|
||||
}
|
||||
|
||||
if (protocol && /^\/\//.test(urlValue)) {
|
||||
return `${protocol}:${urlValue}`;
|
||||
}
|
||||
|
||||
if (origin && /^\//.test(urlValue)) {
|
||||
return `${origin}${urlValue}`;
|
||||
}
|
||||
|
||||
if (origin && /^\.\//.test(urlValue)) {
|
||||
return `${origin}${urlValue.slice(1)}`;
|
||||
}
|
||||
|
||||
if (origin) {
|
||||
return `${origin}/${urlValue}`;
|
||||
}
|
||||
|
||||
return urlValue;
|
||||
}
|
||||
|
||||
function iterateXPathResult(iterator, results = []) {
|
||||
const element = iterator.iterateNext();
|
||||
|
||||
if (element) {
|
||||
return iterateXPathResult(iterator, [...results, element]);
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
function getElements(context, selector, first = false) {
|
||||
if (!selector) {
|
||||
return context;
|
||||
}
|
||||
|
||||
if (/^\/\//.test(selector)) {
|
||||
// XPath selector
|
||||
const iterator = globalWindow.document.evaluate(selector, context, null, globalWindow.XPathResult.ORDERED_NODE_ITERATOR_TYPE, null);
|
||||
|
||||
if (first) {
|
||||
return iterator.iterateNext();
|
||||
}
|
||||
|
||||
return iterateXPathResult(iterator);
|
||||
}
|
||||
|
||||
if (first) {
|
||||
return context.querySelector(selector);
|
||||
}
|
||||
|
||||
return Array.from(context.querySelectorAll(selector));
|
||||
}
|
||||
|
||||
function q(context, selectors, attrArg, applyTrim = true) {
|
||||
if (!selectors && context.nodeName === '#document') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const attr = attrArg === true ? 'textContent' : attrArg;
|
||||
const element = [].concat(selectors).reduce((acc, selector) => acc || getElements(context, selector, true), null);
|
||||
|
||||
if (!element) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (attr) {
|
||||
const value = element[attr] || element.getAttribute(attr);
|
||||
|
||||
return applyTrim && typeof value === 'string' ? trim(value) : value;
|
||||
}
|
||||
|
||||
return element;
|
||||
}
|
||||
|
||||
function all(context, selectors, attrArg, applyTrim = true) {
|
||||
const attr = attrArg === true ? 'textContent' : attrArg;
|
||||
const elements = [].concat(selectors).reduce((acc, selector) => acc || getElements(context, selector), null);
|
||||
|
||||
if (!Array.isArray(elements)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (attr) {
|
||||
return elements.map((el) => q(el, null, attr, applyTrim));
|
||||
}
|
||||
|
||||
return elements;
|
||||
}
|
||||
|
||||
function exists(context, selector) {
|
||||
return !!q(context, selector);
|
||||
}
|
||||
|
||||
function count(context, selector) {
|
||||
return all(context, selector)?.length || 0;
|
||||
}
|
||||
|
||||
function content(context, selector, applyTrim = true) {
|
||||
return q(context, selector, 'textContent', applyTrim);
|
||||
}
|
||||
|
||||
function contents(context, selector, applyTrim) {
|
||||
return all(context, selector, 'textContent', applyTrim);
|
||||
}
|
||||
|
||||
function html(context, selector) {
|
||||
const el = q(context, selector, null, true);
|
||||
|
||||
return el && el.innerHTML;
|
||||
}
|
||||
|
||||
function htmls(context, selector) {
|
||||
const els = all(context, selector, null, true);
|
||||
|
||||
return els.map((el) => el.innerHTML);
|
||||
}
|
||||
|
||||
function execute(context, selector = 'script') {
|
||||
const scripts = htmls(context, selector);
|
||||
const originalGlobal = Object.fromEntries(Object.entries(global));
|
||||
|
||||
const errors = scripts?.reduce((accErrors, script) => {
|
||||
try {
|
||||
Function(script)(); /* eslint-disable-line no-new-func */
|
||||
|
||||
return accErrors;
|
||||
} catch (error) {
|
||||
// the script failed
|
||||
return [...accErrors, error];
|
||||
}
|
||||
}, []);
|
||||
|
||||
const data = Object.fromEntries(Object.entries(global).filter(([key, value]) => {
|
||||
if (originalGlobal[key] !== value) {
|
||||
delete global[key];
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}));
|
||||
|
||||
return {
|
||||
...data,
|
||||
errors,
|
||||
};
|
||||
}
|
||||
|
||||
function json(context, selector) {
|
||||
const el = q(context, selector, null, true);
|
||||
|
||||
try {
|
||||
return JSON.parse(el?.innerHTML);
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function jsons(context, selector) {
|
||||
const els = all(context, selector, null, true);
|
||||
|
||||
return els.map((el) => {
|
||||
try {
|
||||
return JSON.parse(el?.innerHTML);
|
||||
} catch (error) {
|
||||
return null;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function texts(context, selector, applyTrim = true, filter = true) {
|
||||
const el = q(context, selector, null, applyTrim);
|
||||
if (!el) return null;
|
||||
|
||||
const nodes = Array.from(el.childNodes)
|
||||
.filter((node) => node.nodeName === '#text')
|
||||
.map((node) => (applyTrim ? trim(node.textContent) : node.textContent));
|
||||
|
||||
return filter ? nodes.filter(Boolean) : nodes;
|
||||
}
|
||||
|
||||
function text(context, selector, applyTrim = true) {
|
||||
const nodes = texts(context, selector, applyTrim, true);
|
||||
if (!nodes) return null;
|
||||
|
||||
const textValue = nodes.join(' ');
|
||||
|
||||
return applyTrim ? trim(textValue) : textValue;
|
||||
}
|
||||
|
||||
function removeStyleFunctionSpaces(el) {
|
||||
// jsdom appears to have a bug where it ignores inline CSS attributes set to a function() containing spaces, e.g. url( image.png )
|
||||
el.setAttribute('style', el.getAttribute('style').replace(/\(\s+(.*)\s+\)/g, (match, cssArgs) => `(${cssArgs})`));
|
||||
}
|
||||
|
||||
function style(context, selector, styleAttr) {
|
||||
const el = q(context, selector);
|
||||
|
||||
if (el?.hasAttribute('style')) {
|
||||
removeStyleFunctionSpaces(el);
|
||||
|
||||
return styleAttr ? el.style[styleAttr] : el.style;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function styles(context, selector, styleAttr) {
|
||||
const elStyles = Array.from(context.querySelectorAll(selector), (el) => {
|
||||
removeStyleFunctionSpaces(el);
|
||||
|
||||
return styleAttr ? el.style[styleAttr] : el.style;
|
||||
});
|
||||
|
||||
return elStyles;
|
||||
}
|
||||
|
||||
function number(context, selector, match = /\d+(\.\d*)?/, attr = 'textContent', matchIndex = 0) {
|
||||
const value = q(context, selector, attr);
|
||||
|
||||
if (value && match) {
|
||||
return Number(value.match(match)?.[matchIndex]);
|
||||
}
|
||||
|
||||
if (value) {
|
||||
return Number(value);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function meta(context, selector, attrArg = 'content', applyTrim = true) {
|
||||
if (/meta\[.*\]/.test(selector)) {
|
||||
return q(context, selector, attrArg, applyTrim);
|
||||
}
|
||||
|
||||
return q(context, `meta[${selector}]`, attrArg, applyTrim);
|
||||
}
|
||||
|
||||
function date(context, selector, format, match, attr = 'textContent') {
|
||||
const dateString = q(context, selector, attr, true);
|
||||
|
||||
if (!dateString) return null;
|
||||
|
||||
return extractDate(dateString, format, match);
|
||||
}
|
||||
|
||||
function dateAgo(context, selector, match = /(\d+)\s*(\w+)/, attr = 'textContent') {
|
||||
const timeString = q(context, selector, attr, 'textContent');
|
||||
|
||||
if (!timeString) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const timeMatch = timeString.match(match);
|
||||
|
||||
if (timeMatch) {
|
||||
const [n, period] = timeMatch.slice(1);
|
||||
const thenDate = moment.utc().subtract(Number(n), period);
|
||||
|
||||
return {
|
||||
date: thenDate.toDate(),
|
||||
precision: period.replace(/s$/, ''),
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function image(context, selector = 'img', attr, { origin, protocol = 'https' } = {}) {
|
||||
const imageEl = (attr && q(context, selector, attr))
|
||||
|| q(context, selector, 'data-src')
|
||||
|| q(context, selector, 'src');
|
||||
|
||||
return prefixUrl(imageEl, origin, protocol);
|
||||
}
|
||||
|
||||
function images(context, selector = 'img', attr, { origin, protocol = 'https' } = {}) {
|
||||
const attribute = attr
|
||||
|| (q(context, selector, 'data-src') && 'data-src')
|
||||
|| (q(context, selector, 'src') && 'src');
|
||||
|
||||
const imageEls = all(context, selector, attribute);
|
||||
|
||||
return imageEls.map((imageEl) => prefixUrl(imageEl, origin, protocol));
|
||||
}
|
||||
|
||||
function url(context, selector = 'a', attr = 'href', { origin, protocol = 'https', object = false } = {}) {
|
||||
const urlEl = q(context, selector, attr);
|
||||
const prefixedUrl = prefixUrl(urlEl, origin, protocol);
|
||||
|
||||
if (prefixedUrl && object) {
|
||||
return new URL(prefixedUrl);
|
||||
}
|
||||
|
||||
return prefixedUrl;
|
||||
}
|
||||
|
||||
function urls(context, selector = 'a', attr = 'href', { origin, protocol = 'https' } = {}) {
|
||||
const urlEls = all(context, selector, attr);
|
||||
|
||||
return attr ? urlEls.map((urlEl) => prefixUrl(urlEl, origin, protocol)) : urlEls;
|
||||
}
|
||||
|
||||
function sourceSet(context, selector, attr = 'srcset', options = {}) {
|
||||
const srcset = q(context, selector, attr);
|
||||
|
||||
if (!srcset) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const sources = srcset
|
||||
.split(/\s*,\s*/)
|
||||
.map((source) => {
|
||||
const [link, descriptor] = source.split(' ');
|
||||
|
||||
if (link) {
|
||||
return {
|
||||
descriptor: descriptor || 'fallback',
|
||||
url: prefixUrl(link, options.origin, options.protocol),
|
||||
};
|
||||
}
|
||||
|
||||
return null;
|
||||
})
|
||||
.filter(Boolean)
|
||||
.sort((sourceA, sourceB) => {
|
||||
if (sourceB.descriptor === 'fallback' || parseInt(sourceA.descriptor, 10) > parseInt(sourceB.descriptor, 10)) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (parseInt(sourceA.descriptor, 10) < parseInt(sourceB.descriptor, 10)) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
});
|
||||
|
||||
if (options.includeDescriptor) {
|
||||
return sources;
|
||||
}
|
||||
|
||||
return sources.map((source) => source.url);
|
||||
}
|
||||
|
||||
function poster(context, selector = 'video', attr = 'poster', { origin, protocol = 'https' } = {}) {
|
||||
const posterEl = q(context, selector, attr);
|
||||
|
||||
return attr ? prefixUrl(posterEl, origin, protocol) : posterEl;
|
||||
}
|
||||
|
||||
function video(context, selector = 'source', attr = 'src', { origin, protocol = 'https' } = {}) {
|
||||
const trailerEl = q(context, selector, attr);
|
||||
|
||||
return attr ? prefixUrl(trailerEl, origin, protocol) : trailerEl;
|
||||
}
|
||||
|
||||
function videos(context, selector = 'source', attr = 'src', { origin, protocol = 'https' } = {}) {
|
||||
const trailerEls = all(context, selector, attr);
|
||||
|
||||
return attr ? trailerEls.map((trailerEl) => prefixUrl(trailerEl, origin, protocol)) : trailerEls;
|
||||
}
|
||||
|
||||
function duration(context, selector, match, attr = 'textContent') {
|
||||
const durationString = q(context, selector, attr);
|
||||
|
||||
if (!durationString) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const durationMatch = durationToSeconds(durationString, match);
|
||||
|
||||
if (durationMatch) {
|
||||
return durationMatch;
|
||||
}
|
||||
|
||||
const timestampMatch = durationString.match(/(\d+H)?\s*(\d+M)?\s*\d+S?/i);
|
||||
|
||||
if (timestampMatch) {
|
||||
const hours = timestampMatch[0].match(/(\d+)H/i)?.[1] || 0;
|
||||
const minutes = timestampMatch[0].match(/(\d+)M/i)?.[1] || 0;
|
||||
const seconds = timestampMatch[0].match(/(\d+)(S|$)/i)?.[1] || 0;
|
||||
|
||||
return (Number(hours) * 3600) + (Number(minutes) * 60) + Number(seconds);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
const legacyFuncs = {
|
||||
q,
|
||||
qa: all,
|
||||
qall: all,
|
||||
qd: date,
|
||||
qdate: date,
|
||||
qh: html,
|
||||
qhtml: html,
|
||||
qi: image,
|
||||
qimage: image,
|
||||
qimages: images,
|
||||
qis: images,
|
||||
ql: duration,
|
||||
qlength: duration,
|
||||
qm: meta,
|
||||
qmeta: meta,
|
||||
qp: poster,
|
||||
qposter: poster,
|
||||
qs: all,
|
||||
qt: video,
|
||||
qtext: text,
|
||||
qtexts: texts,
|
||||
qtrailer: video,
|
||||
qtrailers: videos,
|
||||
qts: videos,
|
||||
qtx: text,
|
||||
qtxs: texts,
|
||||
qtxt: text,
|
||||
qtxts: texts,
|
||||
// qu: url,
|
||||
qurl: url,
|
||||
qurls: urls,
|
||||
qus: urls,
|
||||
};
|
||||
|
||||
const quFuncs = {
|
||||
all,
|
||||
cnt: content,
|
||||
cnts: contents,
|
||||
content,
|
||||
contents,
|
||||
count,
|
||||
date,
|
||||
dateAgo,
|
||||
dur: duration,
|
||||
duration,
|
||||
el: q,
|
||||
element: q,
|
||||
execute,
|
||||
exec: execute,
|
||||
exists,
|
||||
html,
|
||||
htmls,
|
||||
image,
|
||||
images,
|
||||
img: image,
|
||||
imgs: images,
|
||||
json,
|
||||
jsons,
|
||||
length: duration,
|
||||
meta,
|
||||
num: number,
|
||||
number,
|
||||
poster,
|
||||
q,
|
||||
sourceSet,
|
||||
sources: sourceSet,
|
||||
srcs: sourceSet,
|
||||
srcset: sourceSet,
|
||||
style,
|
||||
styles,
|
||||
text,
|
||||
texts,
|
||||
trailer: video,
|
||||
url,
|
||||
urls,
|
||||
video,
|
||||
videos,
|
||||
};
|
||||
|
||||
function init(context, selector, window) {
|
||||
if (!context) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const element = selector ? context.querySelector(selector) : context;
|
||||
|
||||
if (!element) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const legacyContextFuncs = Object.entries(legacyFuncs) // dynamically attach methods with context
|
||||
.reduce((acc, [key, func]) => ({
|
||||
...acc,
|
||||
[key]: (...args) => (args[0] instanceof globalWindow.HTMLElement // allow for different context
|
||||
? func(...args)
|
||||
: func(element, ...args)),
|
||||
}), {});
|
||||
|
||||
const quContextFuncs = Object.entries(quFuncs) // dynamically attach methods with context
|
||||
.reduce((acc, [key, func]) => ({
|
||||
...acc,
|
||||
[key]: (...args) => (args[0]?.nodeType === undefined // allow for different context
|
||||
? func(element, ...args)
|
||||
: func(...args)),
|
||||
}), {});
|
||||
|
||||
return {
|
||||
element,
|
||||
el: element,
|
||||
html: element.outerHTML || element.body.outerHTML,
|
||||
text: trim(element.textContent),
|
||||
...(window && {
|
||||
window,
|
||||
document: window.document,
|
||||
}),
|
||||
...legacyContextFuncs,
|
||||
qu: quContextFuncs,
|
||||
query: quContextFuncs,
|
||||
};
|
||||
}
|
||||
|
||||
function initAll(context, selector, window) {
|
||||
if (Array.isArray(context)) {
|
||||
return context.map((element) => init(element, null, window));
|
||||
}
|
||||
|
||||
return Array.from(context.querySelectorAll(selector))
|
||||
.map((element) => init(element, null, window));
|
||||
}
|
||||
|
||||
function extract(htmlValue, selector, options) {
|
||||
const { window } = new JSDOM(htmlValue, { virtualConsole, ...options });
|
||||
|
||||
return init(window.document, selector, window);
|
||||
}
|
||||
|
||||
function extractAll(htmlValue, selector, options) {
|
||||
const { window } = new JSDOM(htmlValue, { virtualConsole, ...options });
|
||||
|
||||
return initAll(window.document, selector, window);
|
||||
}
|
||||
|
||||
async function request(method = 'get', urlValue, body, selector, headers, options, queryAll = false) {
|
||||
const res = await (method === 'post'
|
||||
? http.post(urlValue, body, { ...options, headers })
|
||||
: http[method](urlValue, {
|
||||
...options,
|
||||
headers,
|
||||
parse: true,
|
||||
}));
|
||||
|
||||
if (res.ok) {
|
||||
const item = queryAll
|
||||
? initAll(res.document, selector, res.window)
|
||||
: init(res.document, selector, res.window);
|
||||
|
||||
return {
|
||||
item,
|
||||
items: all ? item : [item],
|
||||
res,
|
||||
ok: true,
|
||||
status: res.statusCode,
|
||||
...res,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
item: null,
|
||||
items: [],
|
||||
res,
|
||||
ok: false,
|
||||
status: res.statusCode,
|
||||
...res,
|
||||
};
|
||||
}
|
||||
|
||||
async function get(urlValue, selector, headers, options) {
|
||||
return request('get', urlValue, null, selector, headers, options, false);
|
||||
}
|
||||
|
||||
async function post(urlValue, body, selector, headers, options) {
|
||||
return request('post', urlValue, body, selector, headers, options, false);
|
||||
}
|
||||
|
||||
async function getAll(urlValue, selector, headers, options) {
|
||||
return request('get', urlValue, null, selector, headers, options, true);
|
||||
}
|
||||
|
||||
async function postAll(urlValue, body, selector, headers, options) {
|
||||
return request('post', urlValue, body, selector, headers, options, true);
|
||||
}
|
||||
|
||||
function session(headers, options) {
|
||||
return http.session(headers, options);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
extractDate,
|
||||
extract,
|
||||
extractAll,
|
||||
durationToSeconds,
|
||||
init,
|
||||
initAll,
|
||||
formatDate,
|
||||
get,
|
||||
getAll,
|
||||
http,
|
||||
fetch: get,
|
||||
fetchAll: getAll,
|
||||
context: init,
|
||||
contextAll: initAll,
|
||||
ed: extractDate,
|
||||
ex: extract,
|
||||
exa: extractAll,
|
||||
fd: formatDate,
|
||||
parseDate: extractDate,
|
||||
ctx: init,
|
||||
ctxa: initAll,
|
||||
geta: getAll,
|
||||
qu: quFuncs,
|
||||
query: quFuncs,
|
||||
post,
|
||||
postAll,
|
||||
prefixUrl,
|
||||
session,
|
||||
...legacyFuncs,
|
||||
};
|
||||
@@ -1,20 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
const http = require('./http');
|
||||
|
||||
const sleep = 5000;
|
||||
const timeout = 1000;
|
||||
|
||||
async function init() {
|
||||
try {
|
||||
const res = await http.get(`https://httpstat.us/200?sleep=${sleep}`, {
|
||||
timeout,
|
||||
});
|
||||
|
||||
console.log(res.statusCode);
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
}
|
||||
}
|
||||
|
||||
init();
|
||||
@@ -266,6 +266,7 @@ const actors = [
|
||||
{ entity: 'wakeupnfuck', name: 'Abby Lee Brazil', fields: ['avatar', 'nationality'] },
|
||||
{ entity: 'darkkotv', name: 'Aidra Fox', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'ethnicity', 'height', 'weight', 'measurements', 'naturalBoobs', 'hasTattoos', 'hasPiercings'] },
|
||||
{ entity: 'jaxslayher', name: 'Eliza Ibarra', fields: ['avatar', 'description', 'eyes', 'height', 'weight', 'birthPlace', 'measurements', 'hairColor'] },
|
||||
{ entity: 'cosplayground', name: 'Anna Claire Clouds', url: 'https://cosplayground.com/models/anna-claire-clouds?models=Anna%20Claire%20Clouds', fields: ['avatar', 'description', 'dateOfBirth', 'birthPlace', 'height', 'measurements', 'hairColor', 'ethnicity'] },
|
||||
];
|
||||
|
||||
const actorScrapers = scrapers.actors;
|
||||
|
||||
Reference in New Issue
Block a user