DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Merged migrations.

This commit is contained in:
DebaucheryLibrarian 2026-01-29 20:13:32 +01:00
parent 30923f7cda
commit 888fa50d8a
43 changed files with 1190 additions and 2294 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -7,7 +7,7 @@ Use [nvm](https://github.com/creationix/nvm) to install NodeJS v16.8.0 or newer.
`npm install`
### Set up database
Install PostgreSQL, make sure password authentication is enabled (scram-sha-256). Create a database with a fully privileged user, and a visitor user without privileges (they will be provided by the migration).
Install PostgreSQL, make sure password authentication is enabled (scram-sha-256). Create a database with a fully privileged user.
### Configuration
Do not modify `config/default.js`, but instead create a copy at `config/local.js` containing the properties you wish to change. If you have set `NODE_ENV`, copy `assets/js/config/default.js` to `assets/js/config/[environment].js`. After setting up PostgreSQL and configuring the details, run the following commands to create and populate the tables, and build the project:

0
migrations/20240125011700_manticore.js → assets/20240125011700_manticore.js
View File

1888
migrations/20190325001339_releases.js
View File

File diff suppressed because it is too large Load Diff

58
migrations/20230703013413_random_campaign.js
View File

@ -1,58 +0,0 @@
exports.up = async (knex) => {
await knex.schema.alterTable('random_campaign', (table) => {
table.integer('id')
.notNullable()
.references('id')
.inTable('campaigns');
});
await knex.raw(`
CREATE OR REPLACE FUNCTION get_random_campaign(min_ratio decimal default 0, max_ratio decimal default 1000.0) RETURNS random_campaign AS $$
SELECT * FROM (
SELECT DISTINCT ON (CASE WHEN parent_id IS NOT NULL THEN parent_id ELSE entity_id END)
banner_id, url, entity_id, affiliate_id, parent_id, id
FROM (
SELECT
campaigns.*, entities.parent_id as parent_id
FROM campaigns
LEFT JOIN entities ON entities.id = campaigns.entity_id
LEFT JOIN banners ON banners.id = campaigns.banner_id
WHERE banner_id IS NOT NULL
AND ratio >= min_ratio
AND ratio <= max_ratio
ORDER BY RANDOM()
) random_campaigns
) random_banners
ORDER BY RANDOM()
LIMIT 1;
$$ LANGUAGE SQL STABLE;
`);
};
exports.down = async (knex) => {
await knex.schema.alterTable('random_campaign', (table) => {
table.dropColumn('campaign_id');
});
await knex.raw(`
CREATE OR REPLACE FUNCTION get_random_campaign(min_ratio decimal default 0, max_ratio decimal default 1000.0) RETURNS random_campaign AS $$
SELECT * FROM (
SELECT DISTINCT ON (CASE WHEN parent_id IS NOT NULL THEN parent_id ELSE entity_id END)
banner_id, url, entity_id, affiliate_id, parent_id
FROM (
SELECT
campaigns.*, entities.parent_id as parent_id
FROM campaigns
LEFT JOIN entities ON entities.id = campaigns.entity_id
LEFT JOIN banners ON banners.id = campaigns.banner_id
WHERE banner_id IS NOT NULL
AND ratio >= min_ratio
AND ratio <= max_ratio
ORDER BY RANDOM()
) random_campaigns
) random_banners
ORDER BY RANDOM()
LIMIT 1;
$$ LANGUAGE SQL STABLE;
`);
};

47
migrations/20230709231430_release_summaries.js
View File

@ -1,47 +0,0 @@
const config = require('config');
exports.up = async (knex) => {
await knex.raw(`
CREATE MATERIALIZED VIEW releases_summaries AS (
SELECT
releases.id as release_id,
channels.slug as channel_slug,
channels.type as channel_type,
networks.slug as network_slug,
networks.type as network_type,
parent_networks.slug as parent_network_slug,
parent_networks.type as parent_network_type,
studios.showcased IS NOT false
AND (channels.showcased IS NOT false OR COALESCE(studios.showcased, false) = true)
AND (networks.showcased IS NOT false OR COALESCE(channels.showcased, false) = true OR COALESCE(studios.showcased, false) = true)
AS showcased,
batches.showcased AS batch_showcased,
releases.effective_date,
releases.created_at,
array_agg(tags.slug ORDER BY tags.priority DESC) FILTER (WHERE tags.slug IS NOT NULL) AS tags
FROM releases
LEFT JOIN releases_tags ON releases_tags.release_id = releases.id
LEFT JOIN tags ON tags.id = releases_tags.tag_id
LEFT JOIN entities AS channels ON channels.id = releases.entity_id
LEFT JOIN entities AS studios ON studios.id = releases.studio_id
LEFT JOIN entities AS networks ON networks.id = channels.parent_id
LEFT JOIN entities AS parent_networks ON parent_networks.id = networks.parent_id
LEFT JOIN batches ON batches.id = releases.updated_batch_id
GROUP BY releases.id, studios.showcased, batches.showcased,
channels.showcased, channels.slug, channels.type,
networks.showcased, networks.slug, networks.type,
parent_networks.slug, parent_networks.type
);
COMMENT ON MATERIALIZED VIEW releases_summaries IS E'@foreignKey (release_id) references releases (id)';
GRANT ALL ON releases_summaries TO :visitor;
`, {
visitor: knex.raw(config.database.query.user),
});
};
exports.down = async (knex) => {
await knex.raw(`
DROP MATERIALIZED VIEW IF EXISTS releases_summaries;
`);
};

27
migrations/20230723001912_actors_social_entity.js
View File

@ -1,27 +0,0 @@
exports.up = async (knex) => {
await knex.schema.alterTable('actors_social', (table) => {
table.integer('profile_id')
.references('id')
.inTable('actors_profiles');
table.dropUnique(['url', 'actor_id']);
table.unique(['url', 'actor_id', 'profile_id']);
});
await knex.raw(`
CREATE UNIQUE INDEX actors_social_url_actor_id_null_unique ON actors_social (url, actor_id) WHERE profile_id IS NULL;
`);
};
exports.down = async (knex) => {
await knex.raw(`
DROP INDEX actors_social_url_actor_id_null_unique;
`);
await knex.schema.alterTable('actors_social', (table) => {
table.dropUnique(['url', 'actor_id', 'profile_id']);
table.unique(['url', 'actor_id']);
table.dropColumn('profile_id');
});
};

24
migrations/20230725001453_caps.js
View File

@ -1,24 +0,0 @@
const config = require('config');
exports.up = async (knex) => {
await knex.schema.createTable('releases_caps', (table) => {
table.integer('release_id')
.notNullable()
.references('id')
.inTable('releases')
.onDelete('cascade');
table.text('media_id')
.notNullable()
.references('id')
.inTable('media');
});
await knex.raw('GRANT ALL ON releases_caps TO :visitor;', {
visitor: knex.raw(config.database.query.user),
});
};
exports.down = async (knex) => {
await knex.schema.dropTable('releases_caps');
};

27
migrations/20230725020639_blood_type.js
View File

@ -1,27 +0,0 @@
exports.up = async (knex) => {
await knex.schema.alterTable('actors_profiles', (table) => {
table.string('hair_type');
table.decimal('shoe_size');
table.string('blood_type');
});
await knex.schema.alterTable('actors', (table) => {
table.string('hair_type');
table.decimal('shoe_size');
table.string('blood_type');
});
};
exports.down = async (knex) => {
await knex.schema.alterTable('actors_profiles', (table) => {
table.dropColumn('hair_type');
table.dropColumn('shoe_size');
table.dropColumn('blood_type');
});
await knex.schema.alterTable('actors', (table) => {
table.dropColumn('hair_type');
table.dropColumn('shoe_size');
table.dropColumn('blood_type');
});
};

13
migrations/20230725031547_stealth_batch.js
View File

@ -1,13 +0,0 @@
exports.up = async (knex) => {
await knex.schema.alterTable('batches', (table) => {
table.boolean('showcased')
.notNullable()
.defaultTo(true);
});
};
exports.down = async (knex) => {
await knex.schema.alterTable('batches', (table) => {
table.dropColumn('showcased');
});
};

27
migrations/20230801225038_scene_titles.js
View File

@ -1,27 +0,0 @@
exports.up = async (knex) => {
await knex.schema.alterTable('releases', (table) => {
table.specificType('alt_titles', 'text ARRAY');
});
await knex.schema.alterTable('movies', (table) => {
table.specificType('alt_titles', 'text ARRAY');
});
await knex.schema.alterTable('series', (table) => {
table.specificType('alt_titles', 'text ARRAY');
});
};
exports.down = async (knex) => {
await knex.schema.alterTable('releases', (table) => {
table.dropColumn('alt_titles');
});
await knex.schema.alterTable('movies', (table) => {
table.dropColumn('alt_titles');
});
await knex.schema.alterTable('series', (table) => {
table.dropColumn('alt_titles');
});
};

36
migrations/20230811015424_teasers.js
View File

@ -1,36 +0,0 @@
exports.up = async (knex) => {
await knex.schema.createTable('movies_teasers', (table) => {
table.integer('movie_id', 16)
.notNullable()
.references('id')
.inTable('movies')
.onDelete('cascade');
table.text('media_id', 21)
.notNullable()
.references('id')
.inTable('media');
table.unique('movie_id');
});
await knex.schema.createTable('series_teasers', (table) => {
table.integer('serie_id', 16)
.notNullable()
.references('id')
.inTable('series')
.onDelete('cascade');
table.text('media_id', 21)
.notNullable()
.references('id')
.inTable('media');
table.unique('serie_id');
});
};
exports.down = async (knex) => {
await knex.schema.dropTable('movies_teasers');
await knex.schema.dropTable('series_teasers');
};

19
migrations/20231107044032_photocount.js
View File

@ -1,19 +0,0 @@
exports.up = async (knex) => {
await knex.schema.alterTable('releases', (table) => {
table.integer('photo_count');
});
await knex.schema.alterTable('movies', (table) => {
table.integer('photo_count');
});
};
exports.down = async (knex) => {
await knex.schema.alterTable('releases', (table) => {
table.dropColumn('photo_count');
});
await knex.schema.alterTable('movies', (table) => {
table.dropColumn('photo_count');
});
};

42
migrations/20231109011513_alert_andor_pattern.js
View File

@ -1,42 +0,0 @@
const config = require('config');
exports.up = async (knex) => {
await knex.schema.alterTable('alerts', (table) => {
table.boolean('all')
.defaultTo(true);
});
await knex.schema.alterTable('alerts_entities', (table) => {
table.dropUnique('alert_id');
});
await knex.schema.createTable('alerts_matches', (table) => {
table.increments('id');
table.integer('alert_id')
.references('id')
.inTable('alerts')
.onDelete('cascade');
table.string('property');
table.string('expression');
});
await knex.raw(`
GRANT SELECT ON alerts_matches TO :visitor;
`, {
visitor: knex.raw(config.database.query.user),
});
};
exports.down = async (knex) => {
await knex.schema.alterTable('alerts', (table) => {
table.dropColumn('all');
});
await knex.schema.alterTable('alerts_entities', (table) => {
table.unique('alert_id');
});
await knex.schema.dropTable('alerts_matches');
};

21
migrations/20231201040310_tagsplit.js
View File

@ -1,21 +0,0 @@
exports.up = async (knex) => {
await knex.schema.alterTable('tags', (table) => {
table.specificType('implied_tag_ids', 'integer[]');
});
await knex.schema.alterTable('releases_tags', (table) => {
table.enum('source', ['scraper', 'editor', 'implied'])
.notNullable()
.defaultTo('scraper');
});
};
exports.down = async (knex) => {
await knex.schema.alterTable('tags', (table) => {
table.dropColumn('implied_tag_ids');
});
await knex.schema.alterTable('releases_tags', (table) => {
table.dropColumn('source');
});
};

19
migrations/20231216015820_movies_series_photo_count.js
View File

@ -1,19 +0,0 @@
exports.up = async (knex) => {
await knex.schema.alterTable('movies', (table) => {
table.integer('photo_count');
});
await knex.schema.alterTable('series', (table) => {
table.integer('photo_count');
});
};
exports.down = async (knex) => {
await knex.schema.alterTable('movies', (table) => {
table.dropColumn('photo_count');
});
await knex.schema.alterTable('series', (table) => {
table.dropColumn('photo_count');
});
};

62
migrations/20240104234936_actors_metadata.js
View File

@ -1,62 +0,0 @@
const config = require('config');
exports.up = async function up(knex) {
await knex.raw(`
CREATE MATERIALIZED VIEW actors_meta AS (
SELECT
actors.id as actor_id,
COUNT(DISTINCT stashes_actors)::integer as stashed,
COUNT(DISTINCT releases_actors)::integer as scenes,
row_to_json(avatars) as avatar
FROM actors
LEFT JOIN stashes_actors ON stashes_actors.actor_id = actors.id
LEFT JOIN releases_actors ON releases_actors.actor_id = actors.id
LEFT JOIN media AS avatars ON avatars.id = actors.avatar_media_id
GROUP BY
actors.id,
avatars.id
);
CREATE MATERIALIZED VIEW scenes_meta AS (
SELECT
releases.id as scene_id,
COUNT(DISTINCT stashes_scenes)::integer as stashed
FROM releases
LEFT JOIN stashes_scenes ON stashes_scenes.scene_id = releases.id
GROUP BY releases.id
);
CREATE MATERIALIZED VIEW movies_meta AS (
SELECT
movie_id,
stashed,
stashed_scenes,
stashed + stashed_scenes as stashed_total
FROM (
SELECT
movies.id as movie_id,
COUNT(DISTINCT stashes_movies)::integer as stashed,
COUNT(DISTINCT stashes_scenes)::integer as stashed_scenes
FROM movies
LEFT JOIN stashes_movies ON stashes_movies.movie_id = movies.id
LEFT JOIN movies_scenes ON movies_scenes.movie_id = movies.id
LEFT JOIN stashes_scenes ON stashes_scenes.scene_id = movies_scenes.scene_id
GROUP BY movies.id
) AS meta
);
GRANT ALL ON actors_meta TO :visitor;
GRANT ALL ON scenes_meta TO :visitor;
GRANT ALL ON movies_meta TO :visitor;
`, {
visitor: knex.raw(config.database.query.user),
});
};
exports.down = async function down(knex) {
await knex.raw(`
DROP MATERIALIZED VIEW IF EXISTS actors_meta;
DROP MATERIALIZED VIEW IF EXISTS scenes_meta;
DROP MATERIALIZED VIEW IF EXISTS movies_meta;
`);
};

21
migrations/20240229054247_users_case.js
View File

@ -1,21 +0,0 @@
exports.up = async (knex) => {
await knex.schema.alterTable('users', (table) => {
table.dropUnique('username');
});
await knex.raw(`
CREATE UNIQUE INDEX username_unique_index ON users (LOWER(username));
CREATE UNIQUE INDEX email_unique_index ON users (LOWER(email));
`);
};
exports.down = async (knex) => {
await knex.raw(`
DROP INDEX IF EXISTS username_unique_index;
DROP INDEX IF EXISTS email_unique_index;
`);
await knex.schema.alterTable('users', (table) => {
table.unique('username');
});
};

22
migrations/20240303020146_stashes_meta.js
View File

@ -1,22 +0,0 @@
exports.up = async function up(knex) {
await knex.raw(`
CREATE MATERIALIZED VIEW stashes_meta AS (
SELECT
stashes.id as stash_id,
COUNT(DISTINCT stashes_scenes)::integer as stashed_scenes,
COUNT(DISTINCT stashes_movies)::integer as stashed_movies,
COUNT(DISTINCT stashes_actors)::integer as stashed_actors
FROM stashes
LEFT JOIN stashes_scenes ON stashes_scenes.stash_id = stashes.id
LEFT JOIN stashes_movies ON stashes_movies.stash_id = stashes.id
LEFT JOIN stashes_actors ON stashes_actors.stash_id = stashes.id
GROUP BY stashes.id
);
`);
};
exports.down = async function down(knex) {
await knex.raw(`
DROP MATERIALIZED VIEW IF EXISTS stashes_meta;
`);
};

38
migrations/20240414003818_alerts_any.js
View File

@ -1,38 +0,0 @@
exports.up = async (knex) => {
await knex.schema.alterTable('alerts', (table) => {
table.boolean('all_actors')
.notNullable()
.defaultTo(true);
table.boolean('all_entities')
.notNullable()
.defaultTo(true);
table.boolean('all_tags')
.notNullable()
.defaultTo(true);
table.boolean('all_matches')
.notNullable()
.defaultTo(true);
});
await knex.raw(`
UPDATE alerts
SET
all_actors = false,
all_entities = false,
all_tags = false,
all_matches= false
WHERE alerts.all = false;
`);
};
exports.down = async (knex) => {
await knex.schema.alterTable('alerts', (table) => {
table.dropColumn('all_actors');
table.dropColumn('all_entities');
table.dropColumn('all_tags');
table.dropColumn('all_matches');
});
};

65
migrations/20240815013526_entities_options.js
View File

@ -1,65 +0,0 @@
const config = require('config');
exports.up = async (knex) => {
await knex.schema.alterTable('entities', (table) => {
// internal options, as opposed to parameters for scraper options
table.json('options');
});
await knex.schema.alterTable('releases', (table) => {
table.dropForeign('entity_id');
table.foreign('entity_id')
.references('id')
.inTable('entities')
.onDelete('cascade');
});
await knex.schema.alterTable('releases_caps', (table) => {
table.unique(['release_id', 'media_id']);
});
await knex.schema.createTable('movies_tags', (table) => {
table.integer('tag_id')
.references('id')
.inTable('tags');
table.integer('movie_id')
.notNullable()
.references('id')
.inTable('movies')
.onDelete('cascade');
table.text('original_tag');
table.text('source')
.defaultTo('scraper');
table.unique(['tag_id', 'movie_id']);
});
await knex.raw('GRANT ALL ON ALL TABLES IN SCHEMA public TO :visitor;', {
visitor: knex.raw(config.database.query.user),
});
};
exports.down = async (knex) => {
await knex.schema.alterTable('entities', (table) => {
table.dropColumn('options');
});
await knex.schema.alterTable('releases', (table) => {
table.dropForeign('entity_id');
table.foreign('entity_id')
.references('id')
.inTable('entities')
.onDelete('no action');
});
await knex.schema.alterTable('releases_caps', (table) => {
table.dropUnique(['release_id', 'media_id']);
});
await knex.schema.dropTable('movies_tags');
};

31
migrations/20240822005656_entity_tags_cascade.js
View File

@ -1,31 +0,0 @@
exports.up = async (knex) => {
await knex.schema.alterTable('entities_tags', (table) => {
table.dropForeign('tag_id');
table.dropForeign('entity_id');
table.foreign('tag_id')
.references('id')
.inTable('tags')
.onDelete('cascade');
table.foreign('entity_id')
.references('id')
.inTable('entities')
.onDelete('cascade');
});
};
exports.down = async (knex) => {
await knex.schema.alterTable('entities_tags', (table) => {
table.dropForeign('tag_id');
table.dropForeign('entity_id');
table.foreign('tag_id')
.references('id')
.inTable('tags');
table.foreign('entity_id')
.references('id')
.inTable('entities');
});
};

25
migrations/20240825232707_summaries.js
View File

@ -1,25 +0,0 @@
exports.up = async (knex) => {
await knex.schema.createTable('users_templates', (table) => {
table.increments('id');
table.integer('user_id')
.notNullable()
.references('id')
.inTable('users');
table.string('name')
.notNullable();
table.text('template')
.notNullable();
table.unique(['user_id', 'name']);
table.datetime('created_at')
.defaultTo(knex.fn.now());
});
};
exports.down = async (knex) => {
await knex.schema.dropTable('users_templates');
};

28
migrations/20240830234355_api_keys.js
View File

@ -1,28 +0,0 @@
exports.up = async (knex) => {
await knex.schema.createTable('users_keys', (table) => {
table.increments('id');
table.integer('user_id')
.notNullable()
.references('id')
.inTable('users');
table.text('key')
.notNullable();
table.string('identifier');
table.unique(['user_id', 'identifier']);
table.datetime('last_used_at');
table.specificType('last_used_ip', 'inet');
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
});
};
exports.down = async (knex) => {
await knex.schema.dropTable('users_keys');
};

87
migrations/20240904234305_content_versions.js
View File

@ -1,87 +0,0 @@
exports.up = async (knex) => {
await knex.schema.createTable('scenes_revisions', (table) => {
table.increments('id');
table.integer('scene_id')
.notNullable()
.references('id')
.inTable('releases')
.onDelete('set null');
table.integer('user_id')
.references('id')
.inTable('users')
.onDelete('set null');
table.json('base')
.notNullable();
table.json('deltas')
.notNullable();
table.text('hash')
.notNullable();
table.text('comment');
table.boolean('approved');
table.integer('reviewed_by')
.references('id')
.inTable('users')
.onDelete('set null');
table.datetime('reviewed_at');
table.text('feedback');
table.datetime('applied_at');
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
});
await knex.schema.createTable('bans', (table) => {
table.increments('id');
table.integer('user_id')
.references('id')
.inTable('users')
.onDelete('set null');
table.string('username');
table.specificType('ip', 'cidr');
table.boolean('match_all')
.notNullable()
.defaultTo(false);
table.string('scope');
table.boolean('shadow');
table.integer('banned_by')
.references('id')
.inTable('users')
.onDelete('set null');
table.datetime('expires_at')
.notNullable();
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
});
await knex.schema.alterTable('users', (table) => {
table.specificType('last_ip', 'cidr');
});
};
exports.down = async (knex) => {
await knex.schema.dropTable('scenes_revisions');
await knex.schema.dropTable('bans');
await knex.schema.alterTable('users', (table) => {
table.dropColumn('last_ip');
});
};

35
migrations/20241016020256_profile_details.js
View File

@ -1,35 +0,0 @@
exports.up = async (knex) => {
await knex.schema.alterTable('actors', (table) => {
table.integer('leg');
table.integer('foot');
table.integer('thigh');
});
await knex.schema.alterTable('actors_profiles', (table) => {
table.integer('leg');
table.integer('foot');
table.integer('thigh');
});
await knex.schema.alterTable('releases', (table) => {
table.integer('video_count');
});
};
exports.down = async (knex) => {
await knex.schema.alterTable('actors', (table) => {
table.dropColumn('leg');
table.dropColumn('foot');
table.dropColumn('thigh');
});
await knex.schema.alterTable('actors_profiles', (table) => {
table.dropColumn('leg');
table.dropColumn('foot');
table.dropColumn('thigh');
});
await knex.schema.alterTable('releases', (table) => {
table.dropColumn('video_count');
});
};

15
migrations/20241016171648_revision_nullable.js
View File

@ -1,15 +0,0 @@
exports.up = async (knex) => {
await knex.schema.alterTable('scenes_revisions', (table) => {
table.integer('scene_id')
.nullable()
.alter();
});
};
exports.down = async (knex) => {
await knex.schema.alterTable('scenes_revisions', (table) => {
table.integer('scene_id')
.notNullable()
.alter();
});
};

119
migrations/20241019013040_actor_revisions.js
View File

@ -1,119 +0,0 @@
exports.up = async (knex) => {
await knex.raw('CREATE UNIQUE INDEX unique_main_profiles ON actors_profiles (actor_id) WHERE (entity_id IS NULL);');
await knex.schema.createTable('actors_revisions', (table) => {
table.increments('id');
table.integer('actor_id')
.references('id')
.inTable('actors')
.onDelete('set null');
table.integer('profile_id')
.references('id')
.inTable('actors_profiles')
.onDelete('set null');
table.integer('user_id')
.references('id')
.inTable('users')
.onDelete('set null');
table.json('base')
.notNullable();
table.json('deltas')
.notNullable();
table.text('hash')
.notNullable();
table.text('comment');
table.boolean('approved');
table.integer('reviewed_by')
.references('id')
.inTable('users')
.onDelete('set null');
table.datetime('reviewed_at');
table.text('feedback');
table.datetime('applied_at');
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
});
await knex.schema.alterTable('actors', (table) => {
table.integer('boobs_volume');
table.enum('boobs_implant', ['saline', 'silicone', 'gummy', 'fat']);
table.enum('boobs_placement', ['over', 'under']);
table.string('boobs_surgeon');
table.boolean('natural_butt');
table.integer('butt_volume');
table.enum('butt_implant', ['bbl', 'lift', 'silicone', 'lipo', 'filler', 'mms']);
table.boolean('natural_lips');
table.integer('lips_volume');
table.string('agency');
});
await knex.schema.alterTable('actors_profiles', (table) => {
table.integer('boobs_volume');
table.enum('boobs_implant', ['saline', 'silicone', 'gummy', 'fat']);
table.enum('boobs_placement', ['over', 'under']);
table.string('boobs_surgeon');
table.boolean('natural_butt');
table.integer('butt_volume');
table.enum('butt_implant', ['bbl', 'lift', 'silicone', 'lipo', 'filler', 'mms']);
table.boolean('natural_lips');
table.integer('lips_volume');
table.string('agency');
});
};
exports.down = async (knex) => {
await knex.raw('DROP INDEX unique_main_profiles;');
await knex.schema.dropTable('actors_revisions');
await knex.schema.alterTable('actors', (table) => {
table.dropColumn('boobs_volume');
table.dropColumn('boobs_implant');
table.dropColumn('boobs_placement');
table.dropColumn('boobs_surgeon');
table.dropColumn('natural_butt');
table.dropColumn('butt_volume');
table.dropColumn('butt_implant');
table.dropColumn('natural_lips');
table.dropColumn('lips_volume');
table.dropColumn('agency');
});
await knex.schema.alterTable('actors_profiles', (table) => {
table.dropColumn('boobs_volume');
table.dropColumn('boobs_implant');
table.dropColumn('boobs_placement');
table.dropColumn('boobs_surgeon');
table.dropColumn('natural_butt');
table.dropColumn('butt_volume');
table.dropColumn('butt_implant');
table.dropColumn('natural_lips');
table.dropColumn('lips_volume');
table.dropColumn('agency');
});
};

19
migrations/20241024182155_actor_fields.js
View File

@ -1,19 +0,0 @@
function createColumns(table) {
table.enum('boobs_incision', ['mammary', 'areolar', 'crescent', 'lollipop', 'anchor', 'axillary', 'umbilical']);
table.boolean('natural_labia');
}
exports.up = async (knex) => {
await knex.schema.alterTable('actors', createColumns);
await knex.schema.alterTable('actors_profiles', createColumns);
};
function dropColumns(table) {
table.dropColumn('boobs_incision');
table.dropColumn('natural_labia');
}
exports.down = async (knex) => {
await knex.schema.alterTable('actors', dropColumns);
await knex.schema.alterTable('actors_profiles', dropColumns);
};

75
migrations/20241025174046_actors_avatars.js
View File

@ -1,75 +0,0 @@
exports.up = async function(knex) {
// restore avatars in table in case of rollback and rerun
const avatars = await knex('actors_avatars')
.select('actors_avatars.*', 'actors_profiles.actor_id')
.leftJoin('actors_profiles', 'actors_profiles.id', 'actors_avatars.profile_id');
await knex('actors_avatars').delete();
await knex.schema.alterTable('actors_avatars', (table) => {
table.integer('profile_id')
.nullable()
.alter();
table.integer('actor_id')
.notNullable()
.references('id')
.inTable('actors');
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
table.dropUnique('profile_id');
table.unique(['profile_id', 'media_id']);
});
await knex.schema.alterTable('media', (table) => {
// actor avatars often retain the same URL when updated, handle URL-deduping in app code
table.dropUnique('source');
table.string('source_version'); // usually etag
});
await knex.raw('CREATE UNIQUE INDEX unique_main_avatars ON actors_avatars (actor_id) WHERE (profile_id IS NULL);');
if (avatars.length > 0) {
await knex('actors_avatars').insert(avatars);
}
const profiles = await knex('actors_profiles')
.select('id', 'actor_id', 'avatar_media_id')
.whereNotNull('avatar_media_id');
await knex('actors_avatars')
.insert(profiles.map((profile) => ({
actor_id: profile.actor_id,
profile_id: profile.id,
media_id: profile.avatar_media_id,
})))
.onConflict()
.ignore();
};
exports.down = async function(knex) {
// no need to delete all entries, only the ones incompatible with the old scheme
await knex('actors_avatars')
.whereNull('profile_id')
.delete();
await knex.schema.alterTable('actors_avatars', (table) => {
table.integer('profile_id')
.notNullable()
.alter();
table.dropColumn('actor_id');
table.dropColumn('created_at');
table.unique('profile_id');
table.dropUnique(['profile_id', 'media_id']);
});
await knex.schema.alterTable('media', (table) => {
table.dropColumn('source_version');
table.unique('source');
});
};

46
migrations/20241103060644_socials.js
View File

@ -1,46 +0,0 @@
exports.up = async (knex) => {
await knex.schema.alterTable('actors_social', (table) => {
table.dropUnique(['url', 'actor_id', 'profile_id']);
table.dropColumn('profile_id');
table.string('handle');
table.boolean('is_broken')
.notNullable()
.defaultTo(false);
table.datetime('pinged_at');
table.datetime('verified_at');
table.unique(['actor_id', 'platform', 'handle']);
table.unique(['actor_id', 'url']);
});
await knex.raw('ALTER TABLE actors_social ADD CONSTRAINT socials_url_or_handle CHECK (num_nulls(handle, url) = 1);');
await knex.raw('ALTER TABLE actors_social ADD CONSTRAINT socials_handle_and_platform CHECK (num_nulls(platform, handle) = 2 or num_nulls(platform, handle) = 0);');
await knex.schema.renameTable('actors_social', 'actors_socials');
};
exports.down = async (knex) => {
await knex.raw('ALTER TABLE actors_socials DROP CONSTRAINT socials_url_or_handle;');
await knex.raw('ALTER TABLE actors_socials DROP CONSTRAINT socials_handle_and_platform;');
await knex.schema.renameTable('actors_socials', 'actors_social');
await knex.schema.alterTable('actors_social', (table) => {
table.dropUnique(['actor_id', 'platform', 'handle']);
table.dropUnique(['actor_id', 'url']);
table.integer('profile_id')
.references('id')
.inTable('actors_profiles');
table.dropColumn('handle');
table.dropColumn('verified_at');
table.dropColumn('pinged_at');
table.dropColumn('is_broken');
table.unique(['url', 'actor_id', 'profile_id']);
});
};

53
migrations/20250304025013_media_indexes.js
View File

@ -1,53 +0,0 @@
exports.up = async (knex) => {
await knex.schema.alterTable('media', (table) => table.index('sfw_media_id'));
await knex.schema.alterTable('actors_profiles', (table) => table.index('avatar_media_id'));
await knex.schema.alterTable('actors_avatars', (table) => table.index('media_id'));
await knex.schema.alterTable('actors_photos', (table) => table.index('media_id'));
await knex.schema.alterTable('chapters_photos', (table) => table.index('media_id'));
await knex.schema.alterTable('chapters_posters', (table) => table.index('media_id'));
await knex.schema.alterTable('movies_covers', (table) => table.index('media_id'));
await knex.schema.alterTable('movies_photos', (table) => table.index('media_id'));
await knex.schema.alterTable('movies_posters', (table) => table.index('media_id'));
await knex.schema.alterTable('movies_teasers', (table) => table.index('media_id'));
await knex.schema.alterTable('movies_trailers', (table) => table.index('media_id'));
await knex.schema.alterTable('releases_caps', (table) => table.index('media_id'));
await knex.schema.alterTable('releases_covers', (table) => table.index('media_id'));
await knex.schema.alterTable('releases_posters', (table) => table.index('media_id'));
await knex.schema.alterTable('releases_photos', (table) => table.index('media_id'));
await knex.schema.alterTable('releases_teasers', (table) => table.index('media_id'));
await knex.schema.alterTable('releases_trailers', (table) => table.index('media_id'));
await knex.schema.alterTable('series_covers', (table) => table.index('media_id'));
await knex.schema.alterTable('series_photos', (table) => table.index('media_id'));
await knex.schema.alterTable('series_posters', (table) => table.index('media_id'));
await knex.schema.alterTable('series_teasers', (table) => table.index('media_id'));
await knex.schema.alterTable('series_trailers', (table) => table.index('media_id'));
await knex.schema.alterTable('tags_photos', (table) => table.index('media_id'));
await knex.schema.alterTable('tags_posters', (table) => table.index('media_id'));
};
exports.down = async (knex) => {
await knex.schema.alterTable('media', (table) => table.dropIndex('sfw_media_id'));
await knex.schema.alterTable('actors_profiles', (table) => table.dropIndex('avatar_media_id'));
await knex.schema.alterTable('actors_avatars', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('actors_photos', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('chapters_photos', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('chapters_posters', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('movies_covers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('movies_photos', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('movies_posters', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('movies_teasers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('movies_trailers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('releases_caps', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('releases_covers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('releases_posters', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('releases_photos', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('releases_teasers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('releases_trailers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('series_covers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('series_photos', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('series_posters', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('series_teasers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('series_trailers', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('tags_photos', (table) => table.dropIndex('media_id'));
await knex.schema.alterTable('tags_posters', (table) => table.dropIndex('media_id'));
};

102
migrations/20250307030844_quick_alerts.js
View File

@ -1,102 +0,0 @@
const config = require('config');
exports.up = async (knex) => {
await knex.schema.alterTable('alerts', (table) => {
table.boolean('from_preset')
.notNullable()
.defaultTo(false);
table.text('comment');
});
await knex.schema.createMaterializedView('alerts_users_actors', (view) => {
view.columns('user_id', 'actor_id', 'alert_ids');
view.as(
knex('alerts_actors')
.select(
'alerts.user_id',
'alerts_actors.actor_id',
knex.raw('array_agg(distinct alerts.id) as alert_ids'),
knex.raw('(alerts_tags.id is null and alerts_entities.id is null and alerts_matches.id is null and related_actors.id is null) as is_only'),
)
.leftJoin('alerts', 'alerts.id', 'alerts_actors.alert_id')
.leftJoin('alerts_entities', 'alerts_entities.alert_id', 'alerts_actors.alert_id')
.leftJoin('alerts_tags', 'alerts_tags.alert_id', 'alerts_actors.alert_id')
.leftJoin('alerts_matches', 'alerts_matches.alert_id', 'alerts_actors.alert_id')
.leftJoin('alerts_actors as related_actors', (joinBuilder) => {
joinBuilder
.on('related_actors.alert_id', 'alerts_actors.alert_id')
.on('related_actors.actor_id', '!=', 'alerts_actors.actor_id');
})
.groupBy(['user_id', 'alerts_actors.actor_id', 'is_only']),
);
});
await knex.schema.createMaterializedView('alerts_users_tags', (view) => {
view.columns('user_id', 'tag_id', 'alert_ids');
view.as(
knex('alerts_tags')
.select(
'alerts.user_id',
'alerts_tags.tag_id',
knex.raw('array_agg(distinct alerts.id) as alert_ids'),
knex.raw('(alerts_actors.id is null and alerts_entities.id is null and alerts_matches.id is null and related_tags.id is null) as is_only'),
)
.leftJoin('alerts', 'alerts.id', 'alerts_tags.alert_id')
.leftJoin('alerts_entities', 'alerts_entities.alert_id', 'alerts_tags.alert_id')
.leftJoin('alerts_actors', 'alerts_actors.alert_id', 'alerts_tags.alert_id')
.leftJoin('alerts_matches', 'alerts_matches.alert_id', 'alerts_tags.alert_id')
.leftJoin('alerts_tags as related_tags', (joinBuilder) => {
joinBuilder
.on('related_tags.alert_id', 'alerts_tags.alert_id')
.on('related_tags.tag_id', '!=', 'alerts_tags.tag_id');
})
.groupBy(['user_id', 'alerts_tags.tag_id', 'is_only']),
);
});
await knex.schema.createMaterializedView('alerts_users_entities', (view) => {
view.columns('user_id', 'entity_id', 'alert_ids');
view.as(
knex('alerts_entities')
.select(
'alerts.user_id',
'alerts_entities.entity_id',
knex.raw('array_agg(distinct alerts.id) as alert_ids'),
knex.raw('(alerts_actors.id is null and alerts_tags.id is null and alerts_matches.id is null and related_entities.id is null) as is_only'),
)
.leftJoin('alerts', 'alerts.id', 'alerts_entities.alert_id')
.leftJoin('alerts_tags', 'alerts_tags.alert_id', 'alerts_entities.alert_id')
.leftJoin('alerts_actors', 'alerts_actors.alert_id', 'alerts_entities.alert_id')
.leftJoin('alerts_matches', 'alerts_matches.alert_id', 'alerts_entities.alert_id')
.leftJoin('alerts_entities as related_entities', (joinBuilder) => {
joinBuilder
.on('related_entities.alert_id', 'alerts_entities.alert_id')
.on('related_entities.entity_id', '!=', 'alerts_entities.entity_id');
})
.groupBy(['user_id', 'alerts_entities.entity_id', 'is_only']),
);
});
await knex.raw(`
GRANT SELECT ON alerts_users_actors TO :visitor;
GRANT SELECT ON alerts_users_entities TO :visitor;
GRANT SELECT ON alerts_users_tags TO :visitor;
`, {
visitor: knex.raw(config.database.query.user),
});
};
exports.down = async (knex) => {
await knex.schema.alterTable('alerts', (table) => {
table.dropColumn('from_preset');
table.dropColumn('comment');
});
await knex.schema.dropMaterializedView('alerts_users_actors');
await knex.schema.dropMaterializedView('alerts_users_tags');
await knex.schema.dropMaterializedView('alerts_users_entities');
};

21
migrations/20250404051102_metadata.js
View File

@ -1,21 +0,0 @@
exports.up = async function(knex) {
await knex.schema.alterTable('alerts', (table) => {
table.json('meta');
});
await knex.schema.alterTable('stashes', (table) => {
table.text('comment');
table.json('meta');
});
};
exports.down = async function(knex) {
await knex.schema.alterTable('alerts', (table) => {
table.dropColumn('meta');
});
await knex.schema.alterTable('stashes', (table) => {
table.dropColumn('comment');
table.dropColumn('meta');
});
};

22
migrations/20260122033548_affiliate_parameters.js
View File

@ -1,22 +0,0 @@
exports.up = async (knex) => {
await knex('affiliates')
.update('parameters', null);
await knex.schema.alterTable('affiliates', (table) => {
table.dropUnique(['entity_id', 'parameters']);
});
await knex.schema.alterTable('affiliates', (table) => {
table.json('parameters')
.alter();
});
};
exports.down = async (knex) => {
await knex.schema.alterTable('affiliates', (table) => {
table.string('parameters')
.alter();
table.unique(['entity_id', 'parameters']);
});
};

14
migrations/20260123051908_serie_effective_date.js
View File

@ -1,14 +0,0 @@
exports.up = async (knex) => {
await knex.raw(`
/* allow scenes without dates to be mixed inbetween scenes with dates */
ALTER TABLE series
ADD COLUMN effective_date timestamptz
GENERATED ALWAYS AS (COALESCE(date, created_at)) STORED;
`);
};
exports.down = async (knex) => {
await knex.schema.alterTable('series', (table) => {
table.dropColumn('effective_date');
});
};

71
migrations/20260125052234_fingerprints.js
View File

@ -1,71 +0,0 @@
const config = require('config');
exports.up = async (knex) => {
await knex.schema.createTable('fingerprints_types', (table) => {
table.string('type')
.primary();
});
await knex('fingerprints_types').insert([
'oshash',
'phash',
'md5',
'blake2',
].map((type) => ({ type })));
await knex.schema.createTable('releases_fingerprints', (table) => {
table.increments('id');
table.integer('scene_id')
.notNullable()
.references('id')
.inTable('releases');
table.string('hash')
.notNullable()
.index();
table.string('type')
.notNullable()
.references('type')
.inTable('fingerprints_types');
table.integer('duration');
table.integer('width');
table.integer('height');
table.integer('user_id')
.references('id')
.inTable('users');
table.string('source');
table.integer('source_submissions');
table.json('source_meta');
table.integer('batch_id')
.notNullable()
.references('id')
.inTable('batches');
table.datetime('source_created_at');
table.datetime('created_at')
.notNullable()
.defaultTo(knex.fn.now());
});
await knex.raw(`
create unique index scenes_fingerprints_unique
on releases_fingerprints (scene_id, hash, source, user_id)
nulls not distinct
`);
await knex.raw('GRANT ALL ON ALL TABLES IN SCHEMA public TO :visitor;', {
visitor: knex.raw(config.database.query.user),
});
};
exports.down = async function(knex) {
await knex.schema.dropTable('releases_fingerprints');
await knex.schema.dropTable('fingerprints_types');
};

11
migrations/_20230703000556_campaign_html.js
View File

@ -1,11 +0,0 @@
exports.up = async (knex) => {
await knex.schema.alterTable('banners', (table) => {
table.text('html');
});
};
exports.down = async (knex) => {
await knex.schema.alterTable('banners', (table) => {
table.dropColumn('html');
});
};

38
seeds/00_tags.js
View File

@ -1175,8 +1175,8 @@ const tags = [
slug: 'toys',
},
{
name: 'anal toy',
slug: 'anal-toy',
name: 'toy anal',
slug: 'toy-anal',
description: 'Stuffing a toy, such as a dildo or buttplug, into the ass',
},
{
@ -1701,22 +1701,26 @@ const aliases = [
name: 'brunettes',
for: 'brunette',
},
{
name: 'anal toy',
for: 'toy-anal',
},
{
name: 'anal toys',
for: 'anal-toy',
for: 'toy-anal',
},
{
name: 'buttplug',
for: 'anal-toy',
for: 'toy-anal',
secondary: true,
},
{
name: 'butt plug',
for: 'anal-toy',
for: 'toy-anal',
},
{
name: 'butt plugs',
for: 'anal-toy',
for: 'toy-anal',
},
{
name: 'caning',
@ -2203,6 +2207,10 @@ const aliases = [
name: 'p.o.v.',
for: 'pov',
},
{
name: 'pov sex',
for: 'pov',
},
{
name: 'prolapse',
for: 'anal-prolapse',
@ -2392,6 +2400,10 @@ const aliases = [
for: 'tattoos',
secondary: true,
},
{
name: 'tattoo meid',
for: 'tattoos',
},
{
name: 'teens',
for: 'teen',
@ -2425,6 +2437,10 @@ const aliases = [
name: 'trans',
for: 'transsexual',
},
{
name: 'trans sex',
for: 'transsexual',
},
{
name: 'transgender',
for: 'transsexual',
@ -2767,6 +2783,14 @@ const aliases = [
name: 'interviews',
for: 'interview',
},
{
name: 'grote billen',
for: 'big-butt',
},
{
name: 'grote lul',
for: 'big-dick',
},
// censors, amateur allure
{
name: '2 big c---s',
@ -2806,7 +2830,7 @@ const aliases = [
},
{
name: 'b--t p--g',
for: 'anal-toy',
for: 'toy-anal',
},
{
name: 'c--k--g',

8
seeds/02_sites.js
View File

@ -5738,7 +5738,7 @@ const sites = [
name: '5Kporn',
url: 'https://www.5kporn.com',
tags: ['5k'],
parent: '5kvids',
parent: '8kmembers',
parameters: {
siteId: 1,
short: '5KP',
@ -5749,7 +5749,7 @@ const sites = [
name: '5Kteens',
url: 'https://www.5kteens.com',
tags: ['5k'],
parent: '5kvids',
parent: '8kmembers',
parameters: {
siteId: 2,
short: '5KT',
@ -5759,7 +5759,7 @@ const sites = [
slug: '8kmilfs',
name: '8Kmilfs',
url: 'https://www.8kmilfs.com',
parent: '5kvids',
parent: '8kmembers',
parameters: {
siteId: 3,
short: '8KM',
@ -5769,7 +5769,7 @@ const sites = [
slug: '8kteens',
name: '8Kteens',
url: 'https://www.8kteens.com',
parent: '5kvids',
parent: '8kmembers',
parameters: {
siteId: 4,
short: '8KT',

2
seeds/04_media.js
View File

@ -1148,6 +1148,8 @@ exports.seed = (knex) => Promise.resolve()
media_id: mediaIdsByPath[photo.path],
}));
console.log(tagPosterEntries);
await Promise.all([
upsert('tags_posters', tagPosterEntries, 'tag_id', knex),
upsert('tags_photos', tagPhotoEntries, ['tag_id', 'media_id'], knex),

2
seeds/06_affiliates.js
View File

@ -533,7 +533,7 @@ const bannerTags = {
slayed_300_250_vicki_violet: ['lesbian', 'brunette', 'pussy-eating'],
slayed_315_300_ivy_izzy_103388: ['lesbian', 'blonde', 'brunette', 'ass-eating'],
slayed_315_300_vanna_gianna_103313: ['lesbian', 'brunette'],
slayed_728_90_allie_doshi_a: ['lesbian', 'blonde', 'brunette', 'pussy-eating', 'anal-toy'],
slayed_728_90_allie_doshi_a: ['lesbian', 'blonde', 'brunette', 'pussy-eating', 'toy-anal'],
slayed_728_90_ariana_alexis_102779: ['lesbian', 'pussy-eating', 'brunette'],
slayed_728_90_cecilia_scarlit_102783: ['lesbian', 'black', 'brunette', 'strapon'],
slayed_770_76_ivy_izzy_103388: ['lesbian', 'brunette', 'blonde', 'ass-eating'],

200
src/scrapers/bluedonkeymedia.js
View File

@ -1,106 +1,124 @@
'use strict';
const crypto = require('crypto');
const unprint = require('unprint');
const http = require('../utils/http');
async function fetchTrailer(entryId, videoId, channel, { parameters }) {
// query seems superfluous, but mimics native behavior
const url = `https://apiv2.sysero.nl/api/mvh/stream/get/${entryId}/${videoId}?query=(include:(source:()))`;
async function fetchTrailer(entryId, videoId, channel, credentials) {
const url = `https://api.sysero.nl/free-stream?resource_id=${entryId}&video_id=${videoId}`;
const res = await http.get(url, {
const res = await unprint.get(url, {
headers: {
Origin: channel.url,
Credentials: credentials,
Origin: channel.origin,
Credentials: `sysero ${parameters.credentials}`,
},
});
if (res.ok) {
return res.body.data?.attributes.sources.streams.mpd?.url;
return res.data.data?.ITEM?.LINKS?.map((link) => ({
stream: link.URL,
quality: Number(link.HEIGHT) || null,
}));
}
return null;
}
// MVH's slug system seems to break on non-alphanumerical characters, but also supports ID
function getSceneUrl(channel, slug, sceneId) {
if (slug && /^[\w-]+$/i.test(slug)) {
return `${channel.url}/sexfilms/${slug}`;
async function scrapeSceneData(scene, channel, context, isDeep) {
const release = {};
release.entryId = scene.id;
release.url = `${channel.origin}/sexfilms/${scene.slug}`;
release.title = scene.title;
release.description = scene.description;
release.date = unprint.extractDate(scene.active_from, 'YYYY-MM-DD HH:mm:ss');
release.duration = scene.videos?.data?.film?.[0]?.duration;
release.actors = scene.model?.map((model) => ({
entryId: model.id,
name: model.title,
url: model.slug && `${channel.origin}/modellen/${model.slug}`,
}));
release.tags = scene.resources?.data?.filter((tag) => tag.type === 'category').map((tag) => tag.title);
release.language = scene.videos?.data?.film?.[0]?.language;
if (isDeep) {
const thumb = scene.images?.data?.thumb?.[0];
if (thumb) {
release.poster = `https://cdndo.sysero.nl${thumb.path}`;
}
release.photos = scene.images?.data?.album?.map((image) => `https://cdndo.sysero.nl${image.path}`) || [];
const videoId = scene.videos?.data?.trailer?.[0]?.id;
if (context.include.trailers && videoId) {
release.trailer = await fetchTrailer(release.entryId, videoId, channel, context);
}
} else {
[release.poster, ...release.photos] = scene.images?.data?.thumb?.map((image) => `https://cdndo.sysero.nl${image.path}`) || [];
}
return `${channel.url}/sexfilms/${sceneId}`;
if (scene.clips?.data?.[0]) {
release.teaser = `https://cdndo.sysero.nl${scene.clips.data[0].path}`;
}
return release;
}
function scrapeAll(scenes, channel, context) {
return scenes.reduce((acc, scene) => {
const release = {};
return scenes.reduce(async (chain, scene) => {
const acc = await chain;
/*
release.entryId = scene.id;
release.url = getSceneUrl(channel, scene.attributes.slug, scene.id);
release.date = unprint.extractDate(scene.attributes.product.active_from, 'D/M/YY');
release.title = scene.attributes.title;
release.description = scene.attributes.description;
release.duration = unprint.extractDuration(scene.attributes.videos.film?.[0]?.duration);
const posterPath = scene.attributes.images.thumb?.[0]?.path || context.images[scene.id];
const teaserPath = context.clips[scene.relationships.clips?.data[0]?.id];
if (posterPath) {
release.poster = `https://cdndo.sysero.nl${scene.attributes.images.thumb?.[0]?.path || context.images[scene.id]}`;
if (!scene.slug) {
// page 2 of Vurig Vlaanderen and possible others return broken entries, even on website
return acc;
}
if (teaserPath) {
release.teaser = `https://cdndo.sysero.nl${teaserPath}`;
}
release.tags = scene.relationships.categories?.data.map((category) => context.tags[category.id]?.replace(/-/g, ' ')).filter(Boolean);
release.language = scene.attributes.videos.film?.[0]?.language;
const release = await scrapeSceneData(scene, channel, context, false);
if (release.language && channel.parameters.languages && !channel.parameters.languages?.includes(release.language)) {
// all MVH sites list the entire network, but we want to store Flemish scenes under Vurig Vlaanderen
// the international releases should go on MVH, but the API can't filter for NL+EN, so we do it here
return { ...acc, unextracted: [...acc.unextracted, release] };
}
*/
console.log(scene);
console.log(release);
return { ...acc, scenes: [...acc.scenes, release] };
}, {
}, Promise.resolve({
scenes: [],
unextracted: [],
});
}));
}
async function fetchLatest(channel, page, context) {
// query seems to break if any component is left out or even moved
const res = await http.get(`https://apiv2.sysero.nl/api${context.parameters.apiPath}/resources/nl?query=(content:videos,types:(0:video),sort:(published_at:DESC),filters:(active:1,status:published),pagination:(page:${page},per_page:20),include:((resources:(filters:((types:(0:category),status:published)),images:(filters:((types:(0:thumb))))),images:(filters:((types:(0:cover,1:home_cover,2:thumb,3:cover_thumb)))),clips:(),videos:(),categories:())))`, {
// const res = await unprint.get(`https://apiv2.sysero.nl/api${context.parameters.apiPath}/resources/nl?query=(content:videos,types:(0:video),sort:(published_at:DESC),filters:(active:1,status:published),pagination:(page:${page},per_page:20),include:((resources:(filters:((types:(0:category),status:published)),images:(filters:((types:(0:thumb))))),images:(filters:((types:(0:cover,1:home_cover,2:thumb,3:cover_thumb)))),clips:(),videos:(),categories:())))`, {
// const res = await unprint.get(`https://apiv2.sysero.nl/api${context.parameters.apiPath}/resources/nl?query=(content:videos,types:(0:video),sort:(published_at:DESC),filters:(active:1,status:published),pagination:(page:${page},per_page:20),include:((resources:(filters:((types:(0:category),status:published)),images:(filters:((types:(0:thumb))))),images:(filters:((types:(0:cover,1:home_cover,2:thumb,3:cover_thumb)))),clips:(),videos:(),categories:())))`, {
const res = await unprint.get(`https://apiv2.sysero.nl/api${context.parameters.apiPath}/resources/nl?query=(content:videos,types:(0:video),sort:(published_at:DESC),filters:(active:1,status:published),pagination:(page:${page},per_page:20),include:((resources:(filters:((types:(0:category),status:published)),images:(filters:((types:(0:thumb))))),images:(filters:((types:(0:cover,1:home_cover,2:thumb,3:cover_thumb)))),clips:(),videos:(),categories:())))`, {
headers: {
Origin: channel.origin,
Credentials: context.parameters.credentials,
Credentials: `sysero ${context.parameters.credentials}`,
},
});
if (res.ok && res.body.data) {
return scrapeAll(res.body.data, channel, context);
if (res.ok && res.data.data) {
return scrapeAll(res.data.data, channel, context);
}
return res.status;
}
function getCredentials(channel) {
const now = Math.floor(Date.now() / 1000);
async function scrapeScene({ window }, context) {
const data = window.__NUXT__?.state?.resourcesStore?.video;
const hash = crypto
.createHmac('sha256', channel.parameters.secret)
.update(`${channel.parameters.frontend}${now.toString()}`)
.digest('hex');
if (data) {
return scrapeSceneData(data, context.entity, context, true);
}
const credentials = `Syserauth ${channel.parameters.frontend}-${hash}-${now.toString(16)}`;
return credentials;
return null;
}
const falseCountry = /afghanistan/i; // no country defaults to Afghanistan
@ -114,9 +132,11 @@ function getLocation(model) {
.join(', ') || null;
}
function scrapeProfile(model, { entity, includeScenes = true }) {
async function scrapeProfile(model, { entity }) {
const actor = {};
// gender unreliable, seems to report everyone as 'vrouw' (woman)
actor.name = model.title;
actor.url = unprint.prefixUrl(`/modellen/${model.slug}`, entity.url);
@ -136,76 +156,20 @@ function scrapeProfile(model, { entity, includeScenes = true }) {
actor.eyes = model.eye_color;
actor.hairColor = model.hair_color;
if (includeScenes) {
actor.scenes = model.videos?.map((video) => ({
entryId: video.id,
url: getSceneUrl(entity, video.slug, video.id),
title: video.title,
description: video.description,
}));
}
actor.avatar = unprint.prefixUrl(model.images?.[0]?.path, 'https://cdndo.sysero.nl');
actor.scenes = await Promise.all(model.video?.map(async (video) => scrapeSceneData(video, entity, false)));
return actor;
}
function scrapeSceneData(scene, { entity }) {
const release = {};
release.entryId = scene.id;
release.url = getSceneUrl(entity, scene.slug, scene.id);
release.title = scene.title;
release.description = scene.description;
release.date = scene.uploadDate
? new Date(scene.uploadDate)
: unprint.extractDate(scene.product.active_from, 'D/M/YY');
release.actors = scene.models?.map((model) => scrapeProfile(model, { entity, includeScenes: false }));
release.duration = scene.seconds || unprint.extractTimestamp(scene.isoDuration) || Number(scene.video_paid?.duration) * 60;
release.tags = scene.categories?.map((category) => category.slug.replace(/-/g, ' '));
if (scene.thumb) {
release.poster = [
scene.thumb.original,
scene.thumb.xxl,
scene.thumb.xl,
// ... l, m, s, xs, xxs, probably little point trying all of them
].map((poster) => unprint.prefixUrl(poster, 'https://cdndo.sysero.nl'));
}
release.photos = scene.gallery;
if (scene.trailer) {
release.trailer = async () => {
const credentials = getCredentials(entity);
return fetchTrailer(scene.id, scene.trailer.id, entity, credentials);
};
}
return release;
}
function scrapeScene({ _query, window }, context) {
const data = window.__NUXT__?.state?.videoStore?.video;
if (data) {
return scrapeSceneData(data, context);
}
return null;
}
async function fetchProfile(actor, { entity }) {
const credentials = getCredentials(entity);
const url = `${entity.url}/modellen/${actor.slug}`;
async function fetchProfile(actor, { entity, parameters }) {
const url = actor.url || `${entity.url}/modellen/${actor.slug}`;
const res = await unprint.get(url, {
headers: {
Origin: entity.url,
Credentials: credentials,
Origin: entity.origin,
Credentials: `sysero ${parameters.credentials}`,
},
parser: {
runScripts: 'dangerously',

2
src/tags.js
View File

@ -76,7 +76,7 @@ function withRelations(queryBuilder, withMedia) {
async function matchReleaseTags(releases) {
const tags = releases
.map((release) => release.tags).flat()
.map((tag) => tag?.trim().toLowerCase())
.map((tag) => tag?.trim().match(/[a-z0-9]+/ig)?.join(' ').toLowerCase())
.filter(Boolean);
const tagEntries = await knex('tags')