diff --git a/README.md b/README.md index 6237ab13..c44bff0f 100755 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Use [nvm](https://github.com/creationix/nvm) to install NodeJS v16.8.0 or newer. `npm install` ### Set up database -Install PostgreSQL, make sure password authentication is enabled (scram-sha-256). Create a database with a fully privileged user, and a visitor user without privileges (they will be provided by the migration). +Install PostgreSQL, make sure password authentication is enabled (scram-sha-256). Create a database with a fully privileged user. ### Configuration Do not modify `config/default.js`, but instead create a copy at `config/local.js` containing the properties you wish to change. If you have set `NODE_ENV`, copy `assets/js/config/default.js` to `assets/js/config/[environment].js`. After setting up PostgreSQL and configuring the details, run the following commands to create and populate the tables, and build the project: diff --git a/migrations/20240125011700_manticore.js b/assets/20240125011700_manticore.js similarity index 100% rename from migrations/20240125011700_manticore.js rename to assets/20240125011700_manticore.js diff --git a/migrations/20190325001339_releases.js b/migrations/20190325001339_releases.js index e76b639c..764eb687 100755 --- a/migrations/20190325001339_releases.js +++ b/migrations/20190325001339_releases.js @@ -1,12 +1,10 @@ -const config = require('config'); - -exports.up = (knex) => Promise.resolve() - .then(() => knex.schema.createTable('countries', (table) => { - table.text('alpha2', 2) +exports.up = async (knex) => { + await knex.schema.createTable('countries', (table) => { + table.string('alpha2', 2) .unique() .primary(); - table.text('alpha3', 3) + table.string('alpha3', 3) .unique(); table.text('name') @@ -19,22 +17,22 @@ exports.up = (knex) => Promise.resolve() table.integer('priority', 2) .defaultTo(0); - })) - .then(() => knex.schema.createTable('entities_types', (table) => { + }); + + await knex.schema.createTable('entities_types', (table) => { table.text('type') .primary(); - })) - .then(() => { // eslint-disable-line arrow-body-style - // allow vim fold - return knex('entities_types').insert([ - { type: 'network' }, - { type: 'channel' }, - { type: 'studio' }, - { type: 'info' }, - ]); - }) - .then(() => knex.schema.createTable('entities', (table) => { - table.increments('id', 12); + }); + + await knex('entities_types').insert([ + { type: 'network' }, + { type: 'channel' }, + { type: 'studio' }, + { type: 'info' }, + ]); + + await knex.schema.createTable('entities', (table) => { + table.increments('id'); table.integer('parent_id', 12) .references('id') @@ -56,7 +54,9 @@ exports.up = (knex) => Promise.resolve() table.text('url'); table.text('description'); + table.json('parameters'); + table.json('options'); table.integer('priority', 3) .defaultTo(0); @@ -73,8 +73,9 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('media', (table) => { + }); + + await knex.schema.createTable('media', (table) => { table.text('id', 21) .primary(); @@ -99,6 +100,7 @@ exports.up = (knex) => Promise.resolve() table.text('source', 2100); table.text('source_page', 2100); + table.string('source_version'); // usually etag table.text('scraper', 32); table.text('credit', 100); @@ -119,30 +121,30 @@ exports.up = (knex) => Promise.resolve() .defaultTo(false); table.unique('hash'); - table.unique('source'); table.datetime('created_at') .defaultTo(knex.fn.now()); - })) - .then(() => { // eslint-disable-line arrow-body-style - // allow vim fold - return knex.raw(` - CREATE FUNCTION get_random_sfw_media_id() RETURNS varchar AS $$ - SELECT id FROM media - WHERE is_sfw = true - ORDER BY random() - LIMIT 1; - $$ LANGUAGE sql STABLE; - `); - }) - .then(() => knex.schema.alterTable('media', (table) => { + }); + + await knex.raw(` + CREATE FUNCTION get_random_sfw_media_id() RETURNS varchar AS $$ + SELECT id FROM media + WHERE is_sfw = true + ORDER BY random() + LIMIT 1; + $$ LANGUAGE sql STABLE; + `); + + await knex.schema.alterTable('media', (table) => { table.text('sfw_media_id', 21) + .index() .references('id') .inTable('media') .defaultTo(knex.raw('get_random_sfw_media_id()')); - })) - .then(() => knex.schema.createTable('tags_groups', (table) => { - table.increments('id', 12); + }); + + await knex.schema.createTable('tags_groups', (table) => { + table.increments('id'); table.text('name', 32); table.text('description'); @@ -152,9 +154,10 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('tags', (table) => { - table.increments('id', 12); + }); + + await knex.schema.createTable('tags', (table) => { + table.increments('id'); table.text('name'); table.text('description'); @@ -179,53 +182,63 @@ exports.up = (knex) => Promise.resolve() table.text('slug', 32) .unique(); + table.specificType('implied_tag_ids', 'integer[]'); + table.datetime('created_at') .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('tags_posters', (table) => { + }); + + await knex.schema.createTable('tags_posters', (table) => { table.integer('tag_id', 12) .notNullable() .references('id') .inTable('tags'); table.text('media_id', 21) + .index() .notNullable() .references('id') .inTable('media'); table.unique('tag_id'); - })) - .then(() => knex.schema.createTable('tags_photos', (table) => { + }); + + await knex.schema.createTable('tags_photos', (table) => { table.integer('tag_id', 12) .notNullable() .references('id') .inTable('tags'); table.text('media_id', 21) + .index() .notNullable() .references('id') .inTable('media'); table.unique(['tag_id', 'media_id']); - })) - .then(() => knex.schema.createTable('entities_tags', (table) => { + }); + + await knex.schema.createTable('entities_tags', (table) => { table.integer('tag_id', 12) .notNullable() .references('id') - .inTable('tags'); + .inTable('tags') + .onDelete('cascade'); table.integer('entity_id', 12) .notNullable() .references('id') - .inTable('entities'); + .inTable('entities') + .onDelete('cascade'); table.boolean('inherit') .defaultTo(false); table.unique(['tag_id', 'entity_id']); - })) - .then(() => knex.schema.createTable('entities_social', (table) => { - table.increments('id', 16); + }); + + await knex.schema.createTable('entities_social', (table) => { + table.increments('id'); table.text('url'); table.text('platform'); @@ -239,16 +252,22 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('batches', (table) => { - table.increments('id', 12); + }); + + await knex.schema.createTable('batches', (table) => { + table.increments('id'); table.text('comment'); + table.boolean('showcased') + .notNullable() + .defaultTo(true); + table.datetime('created_at') .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('actors', (table) => { - table.increments('id', 12); + }); + + await knex.schema.createTable('actors', (table) => { + table.increments('id'); table.text('name') .notNullable(); @@ -303,14 +322,40 @@ exports.up = (knex) => Promise.resolve() table.integer('height', 3); table.integer('weight', 3); table.text('eyes'); + table.text('hair_color'); table.text('hair_length'); + table.string('hair_type'); table.boolean('has_tattoos'); table.boolean('has_piercings'); table.text('piercings'); table.text('tattoos'); + table.decimal('shoe_size'); + table.integer('leg'); + table.integer('foot'); + table.integer('thigh'); + + table.string('blood_type'); + + table.integer('boobs_volume'); + table.enum('boobs_implant', ['saline', 'silicone', 'gummy', 'fat']); + table.enum('boobs_placement', ['over', 'under']); + table.string('boobs_surgeon'); + table.enum('boobs_incision', ['mammary', 'areolar', 'crescent', 'lollipop', 'anchor', 'axillary', 'umbilical']); + + table.boolean('natural_butt'); + table.integer('butt_volume'); + table.enum('butt_implant', ['bbl', 'lift', 'silicone', 'lipo', 'filler', 'mms']); + + table.boolean('natural_lips'); + table.integer('lips_volume'); + + table.boolean('natural_labia'); + + table.string('agency'); + table.text('avatar_media_id', 21) .references('id') .inTable('media'); @@ -325,9 +370,10 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('actors_profiles', (table) => { - table.increments('id', 12); + }); + + await knex.schema.createTable('actors_profiles', (table) => { + table.increments('id'); table.integer('actor_id', 12) .notNullable() @@ -382,16 +428,43 @@ exports.up = (knex) => Promise.resolve() table.integer('height', 3); table.integer('weight', 3); + table.text('eyes'); table.text('hair_color'); table.text('hair_length'); + table.string('hair_type'); table.boolean('has_tattoos'); table.boolean('has_piercings'); table.text('piercings'); table.text('tattoos'); + table.decimal('shoe_size'); + table.integer('leg'); + table.integer('foot'); + table.integer('thigh'); + + table.string('blood_type'); + + table.integer('boobs_volume'); + table.enum('boobs_implant', ['saline', 'silicone', 'gummy', 'fat']); + table.enum('boobs_placement', ['over', 'under']); + table.string('boobs_surgeon'); + table.enum('boobs_incision', ['mammary', 'areolar', 'crescent', 'lollipop', 'anchor', 'axillary', 'umbilical']); + + table.boolean('natural_butt'); + table.integer('butt_volume'); + table.enum('butt_implant', ['bbl', 'lift', 'silicone', 'lipo', 'filler', 'mms']); + + table.boolean('natural_lips'); + table.integer('lips_volume'); + + table.boolean('natural_labia'); + + table.string('agency'); + table.text('avatar_media_id', 21) + .index() .references('id') .inTable('media'); @@ -403,133 +476,135 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('body', (table) => { + }); + + await knex.raw('CREATE UNIQUE INDEX unique_main_profiles ON actors_profiles (actor_id) WHERE (entity_id IS NULL);'); + + await knex.schema.createTable('body', (table) => { table.text('slug', 20) .primary(); table.text('name'); - })) - .then(() => { // eslint-disable-line arrow-body-style - // allow vim fold - return knex('body').insert([ - // head - { slug: 'head', name: 'head' }, - { slug: 'face', name: 'face' }, - { slug: 'scalp', name: 'scalp' }, - { slug: 'forehead', name: 'forehead' }, - { slug: 'temple', name: 'temple' }, - { slug: 'cheek', name: 'cheek' }, - { slug: 'jaw', name: 'jaw' }, - { slug: 'chin', name: 'chin' }, - { slug: 'neck', name: 'neck' }, - { slug: 'throat', name: 'throat' }, - // eyes - { slug: 'eyelid', name: 'eyelid' }, - { slug: 'eyeball', name: 'eyeball' }, - { slug: 'eyebrow', name: 'eyebrow' }, - // mouth - { slug: 'tongue', name: 'tongue' }, - { slug: 'lip', name: 'lip' }, - { slug: 'upper-lip', name: 'upper lip' }, - { slug: 'lower-lip', name: 'lower lip' }, - { slug: 'inner-lip', name: 'inner lip' }, - { slug: 'inner-lower-lip', name: 'inner lower lip' }, - { slug: 'inner-upper-lip', name: 'inner upper lip' }, - { slug: 'philtrum', name: 'philtrum' }, - { slug: 'above-lip', name: 'above lip' }, - { slug: 'below-lip', name: 'below lip' }, - // nose - { slug: 'nose', name: 'nose' }, - { slug: 'third-eye', name: 'third eye' }, - { slug: 'bridge', name: 'bridge' }, - { slug: 'nostril', name: 'nostril' }, - { slug: 'septum', name: 'septum' }, - { slug: 'septril', name: 'septril' }, - // ear - { slug: 'ear', name: 'ear' }, - { slug: 'earlobe', name: 'earlobe' }, - { slug: 'helix', name: 'helix' }, - { slug: 'tragus', name: 'tragus' }, - { slug: 'conch', name: 'conch' }, - { slug: 'rook', name: 'rook' }, - { slug: 'behind-ear', name: 'behind ear' }, - // arms - { slug: 'arm', name: 'arm' }, - { slug: 'upper-arm', name: 'upper arm' }, - { slug: 'forearm', name: 'forearm' }, - { slug: 'elbow', name: 'elbow' }, - { slug: 'inner-elbow', name: 'inner elbow' }, - { slug: 'outer-elbow', name: 'outer elbow' }, - // hands - { slug: 'hand', name: 'hand' }, - { slug: 'fingers', name: 'fingers' }, - { slug: 'knuckles', name: 'knuckles' }, - { slug: 'thumb', name: 'thumb' }, - { slug: 'index-finger', name: 'index finger' }, - { slug: 'middle-finger', name: 'middle finger' }, - { slug: 'ring-finger', name: 'ring finger' }, - { slug: 'pinky', name: 'pinky' }, - { slug: 'back-of-hand', name: 'back of hand' }, - { slug: 'inner-wrist', name: 'inner wrist' }, - { slug: 'outer-wrist', name: 'outer wrist' }, - // torso - { slug: 'shoulder', name: 'shoulder' }, - { slug: 'collarbone', name: 'collarbone' }, - { slug: 'chest', name: 'chest' }, - { slug: 'rib-cage', name: 'rib cage' }, - { slug: 'breastbone', name: 'breastbone' }, - { slug: 'underboob', name: 'underboob' }, - { slug: 'sideboob', name: 'sideboob' }, - { slug: 'boob', name: 'boob' }, - { slug: 'nipple', name: 'nipple' }, - { slug: 'abdomen', name: 'abdomen' }, - { slug: 'navel', name: 'navel' }, - { slug: 'pelvis', name: 'pelvis' }, - // back - { slug: 'back', name: 'back' }, - { slug: 'upper-back', name: 'upper back' }, - { slug: 'middle-back', name: 'lower back' }, - { slug: 'lower-back', name: 'lower back' }, - { slug: 'spine', name: 'spine' }, - // bottom - { slug: 'butt', name: 'butt' }, - { slug: 'hip', name: 'hip' }, - { slug: 'anus', name: 'anus' }, - // genitals - { slug: 'pubic-mound', name: 'pubic mound' }, - { slug: 'vagina', name: 'vagina' }, - { slug: 'outer-labia', name: 'outer labia' }, - { slug: 'inner-labia', name: 'inner labia' }, - { slug: 'clitoris', name: 'clitoris' }, - { slug: 'penis', name: 'penis' }, - { slug: 'glans', name: 'glans' }, - { slug: 'foreskin', name: 'foreskin' }, - { slug: 'shaft', name: 'shaft' }, - { slug: 'scrotum', name: 'scrotum' }, - // legs - { slug: 'leg', name: 'leg' }, - { slug: 'groin', name: 'groin' }, - { slug: 'upper-leg', name: 'upper leg' }, - { slug: 'thigh', name: 'thigh' }, - { slug: 'lower-leg', name: 'lower leg' }, - { slug: 'shin', name: 'shin' }, - { slug: 'calf', name: 'calf' }, - { slug: 'knee', name: 'knee' }, - { slug: 'inner-knee', name: 'inner knee' }, - // feet - { slug: 'inner-ankle', name: 'inner ankle' }, - { slug: 'outer-ankle', name: 'outer ankle' }, - { slug: 'foot', name: 'foot' }, - { slug: 'toes', name: 'toes' }, - { slug: 'big-toe', name: 'big toe' }, - { slug: 'index-toe', name: 'index toe' }, - { slug: 'middle-toe', name: 'middle toe' }, - { slug: 'fourth-toe', name: 'fourth toe' }, - { slug: 'little-toe', name: 'little toe' }, - ]); - }) - .then(() => knex.schema.createTable('actors_tattoos', (table) => { + }); + + await knex('body').insert([ + // head + { slug: 'head', name: 'head' }, + { slug: 'face', name: 'face' }, + { slug: 'scalp', name: 'scalp' }, + { slug: 'forehead', name: 'forehead' }, + { slug: 'temple', name: 'temple' }, + { slug: 'cheek', name: 'cheek' }, + { slug: 'jaw', name: 'jaw' }, + { slug: 'chin', name: 'chin' }, + { slug: 'neck', name: 'neck' }, + { slug: 'throat', name: 'throat' }, + // eyes + { slug: 'eyelid', name: 'eyelid' }, + { slug: 'eyeball', name: 'eyeball' }, + { slug: 'eyebrow', name: 'eyebrow' }, + // mouth + { slug: 'tongue', name: 'tongue' }, + { slug: 'lip', name: 'lip' }, + { slug: 'upper-lip', name: 'upper lip' }, + { slug: 'lower-lip', name: 'lower lip' }, + { slug: 'inner-lip', name: 'inner lip' }, + { slug: 'inner-lower-lip', name: 'inner lower lip' }, + { slug: 'inner-upper-lip', name: 'inner upper lip' }, + { slug: 'philtrum', name: 'philtrum' }, + { slug: 'above-lip', name: 'above lip' }, + { slug: 'below-lip', name: 'below lip' }, + // nose + { slug: 'nose', name: 'nose' }, + { slug: 'third-eye', name: 'third eye' }, + { slug: 'bridge', name: 'bridge' }, + { slug: 'nostril', name: 'nostril' }, + { slug: 'septum', name: 'septum' }, + { slug: 'septril', name: 'septril' }, + // ear + { slug: 'ear', name: 'ear' }, + { slug: 'earlobe', name: 'earlobe' }, + { slug: 'helix', name: 'helix' }, + { slug: 'tragus', name: 'tragus' }, + { slug: 'conch', name: 'conch' }, + { slug: 'rook', name: 'rook' }, + { slug: 'behind-ear', name: 'behind ear' }, + // arms + { slug: 'arm', name: 'arm' }, + { slug: 'upper-arm', name: 'upper arm' }, + { slug: 'forearm', name: 'forearm' }, + { slug: 'elbow', name: 'elbow' }, + { slug: 'inner-elbow', name: 'inner elbow' }, + { slug: 'outer-elbow', name: 'outer elbow' }, + // hands + { slug: 'hand', name: 'hand' }, + { slug: 'fingers', name: 'fingers' }, + { slug: 'knuckles', name: 'knuckles' }, + { slug: 'thumb', name: 'thumb' }, + { slug: 'index-finger', name: 'index finger' }, + { slug: 'middle-finger', name: 'middle finger' }, + { slug: 'ring-finger', name: 'ring finger' }, + { slug: 'pinky', name: 'pinky' }, + { slug: 'back-of-hand', name: 'back of hand' }, + { slug: 'inner-wrist', name: 'inner wrist' }, + { slug: 'outer-wrist', name: 'outer wrist' }, + // torso + { slug: 'shoulder', name: 'shoulder' }, + { slug: 'collarbone', name: 'collarbone' }, + { slug: 'chest', name: 'chest' }, + { slug: 'rib-cage', name: 'rib cage' }, + { slug: 'breastbone', name: 'breastbone' }, + { slug: 'underboob', name: 'underboob' }, + { slug: 'sideboob', name: 'sideboob' }, + { slug: 'boob', name: 'boob' }, + { slug: 'nipple', name: 'nipple' }, + { slug: 'abdomen', name: 'abdomen' }, + { slug: 'navel', name: 'navel' }, + { slug: 'pelvis', name: 'pelvis' }, + // back + { slug: 'back', name: 'back' }, + { slug: 'upper-back', name: 'upper back' }, + { slug: 'middle-back', name: 'lower back' }, + { slug: 'lower-back', name: 'lower back' }, + { slug: 'spine', name: 'spine' }, + // bottom + { slug: 'butt', name: 'butt' }, + { slug: 'hip', name: 'hip' }, + { slug: 'anus', name: 'anus' }, + // genitals + { slug: 'pubic-mound', name: 'pubic mound' }, + { slug: 'vagina', name: 'vagina' }, + { slug: 'outer-labia', name: 'outer labia' }, + { slug: 'inner-labia', name: 'inner labia' }, + { slug: 'clitoris', name: 'clitoris' }, + { slug: 'penis', name: 'penis' }, + { slug: 'glans', name: 'glans' }, + { slug: 'foreskin', name: 'foreskin' }, + { slug: 'shaft', name: 'shaft' }, + { slug: 'scrotum', name: 'scrotum' }, + // legs + { slug: 'leg', name: 'leg' }, + { slug: 'groin', name: 'groin' }, + { slug: 'upper-leg', name: 'upper leg' }, + { slug: 'thigh', name: 'thigh' }, + { slug: 'lower-leg', name: 'lower leg' }, + { slug: 'shin', name: 'shin' }, + { slug: 'calf', name: 'calf' }, + { slug: 'knee', name: 'knee' }, + { slug: 'inner-knee', name: 'inner knee' }, + // feet + { slug: 'inner-ankle', name: 'inner ankle' }, + { slug: 'outer-ankle', name: 'outer ankle' }, + { slug: 'foot', name: 'foot' }, + { slug: 'toes', name: 'toes' }, + { slug: 'big-toe', name: 'big toe' }, + { slug: 'index-toe', name: 'index toe' }, + { slug: 'middle-toe', name: 'middle toe' }, + { slug: 'fourth-toe', name: 'fourth toe' }, + { slug: 'little-toe', name: 'little toe' }, + ]); + + await knex.schema.createTable('actors_tattoos', (table) => { table.increments('id'); table.integer('actor_id', 12) @@ -548,8 +623,9 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('actors_piercings', (table) => { + }); + + await knex.schema.createTable('actors_piercings', (table) => { table.increments('id'); table.integer('actor_id', 12) @@ -568,22 +644,35 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('actors_avatars', (table) => { + }); + + await knex.schema.createTable('actors_avatars', (table) => { + table.integer('actor_id') + .notNullable() + .references('id') + .inTable('actors'); + table.integer('profile_id', 12) - .notNullable() .references('id') .inTable('actors_profiles') .onDelete('cascade'); table.text('media_id', 21) + .index() .notNullable() .references('id') .inTable('media'); - table.unique('profile_id'); - })) - .then(() => knex.schema.createTable('actors_photos', (table) => { + table.datetime('created_at') + .notNullable() + .defaultTo(knex.fn.now()); + + table.unique(['profile_id', 'media_id']); + }); + + await knex.raw('CREATE UNIQUE INDEX unique_main_avatars ON actors_avatars (actor_id) WHERE (profile_id IS NULL);'); + + await knex.schema.createTable('actors_photos', (table) => { table.integer('actor_id', 12) .notNullable() .references('id') @@ -591,14 +680,16 @@ exports.up = (knex) => Promise.resolve() .onDelete('cascade'); table.text('media_id', 21) + .index() .notNullable() .references('id') .inTable('media'); table.unique(['actor_id', 'media_id']); - })) - .then(() => knex.schema.createTable('actors_social', (table) => { - table.increments('id', 16); + }); + + await knex.schema.createTable('actors_socials', (table) => { + table.increments('id'); table.text('url'); table.text('platform'); @@ -608,18 +699,33 @@ exports.up = (knex) => Promise.resolve() .references('id') .inTable('actors'); - table.unique(['url', 'actor_id']); + table.string('handle'); + + table.boolean('is_broken') + .notNullable() + .defaultTo(false); table.datetime('created_at') .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('releases', (table) => { - table.increments('id', 16); + + table.datetime('pinged_at'); + table.datetime('verified_at'); + + table.unique(['actor_id', 'platform', 'handle']); + table.unique(['actor_id', 'url']); + }); + + await knex.raw('ALTER TABLE actors_socials ADD CONSTRAINT socials_url_or_handle CHECK (num_nulls(handle, url) = 1);'); + await knex.raw('ALTER TABLE actors_socials ADD CONSTRAINT socials_handle_and_platform CHECK (num_nulls(platform, handle) = 2 or num_nulls(platform, handle) = 0);'); + + await knex.schema.createTable('releases', (table) => { + table.increments('id'); table.integer('entity_id', 12) + .notNullable() .references('id') .inTable('entities') - .notNullable(); + .onDelete('cascade'); table.integer('studio_id', 12) .references('id') @@ -632,6 +738,7 @@ exports.up = (knex) => Promise.resolve() table.text('url', 1000); table.text('title'); + table.specificType('alt_titles', 'text ARRAY'); table.text('slug'); table.timestamp('date'); @@ -656,6 +763,9 @@ exports.up = (knex) => Promise.resolve() table.specificType('qualities', 'text[]'); + table.integer('photo_count'); + table.integer('video_count'); + table.boolean('deep'); table.text('deep_url', 1000); @@ -678,8 +788,9 @@ exports.up = (knex) => Promise.resolve() table.datetime('updated_at') .notNullable() .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('releases_actors', (table) => { + }); + + await knex.schema.createTable('releases_actors', (table) => { table.integer('release_id', 16) .notNullable() .references('id') @@ -701,8 +812,9 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('releases_directors', (table) => { + }); + + await knex.schema.createTable('releases_directors', (table) => { table.integer('release_id', 16) .notNullable() .references('id') @@ -724,8 +836,9 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('releases_posters', (table) => { + }); + + await knex.schema.createTable('releases_posters', (table) => { table.integer('release_id', 16) .notNullable() .references('id') @@ -733,13 +846,15 @@ exports.up = (knex) => Promise.resolve() .onDelete('cascade'); table.text('media_id', 21) + .index() .notNullable() .references('id') .inTable('media'); table.unique('release_id'); - })) - .then(() => knex.schema.createTable('releases_covers', (table) => { + }); + + await knex.schema.createTable('releases_covers', (table) => { table.integer('release_id', 16) .notNullable() .references('id') @@ -747,13 +862,15 @@ exports.up = (knex) => Promise.resolve() .onDelete('cascade'); table.text('media_id', 21) + .index() .notNullable() .references('id') .inTable('media'); table.unique(['release_id', 'media_id']); - })) - .then(() => knex.schema.createTable('releases_trailers', (table) => { + }); + + await knex.schema.createTable('releases_trailers', (table) => { table.integer('release_id', 16) .notNullable() .references('id') @@ -761,13 +878,15 @@ exports.up = (knex) => Promise.resolve() .onDelete('cascade'); table.text('media_id', 21) + .index() .notNullable() .references('id') .inTable('media'); table.unique('release_id'); - })) - .then(() => knex.schema.createTable('releases_teasers', (table) => { + }); + + await knex.schema.createTable('releases_teasers', (table) => { table.integer('release_id', 16) .notNullable() .references('id') @@ -775,13 +894,15 @@ exports.up = (knex) => Promise.resolve() .onDelete('cascade'); table.text('media_id', 21) + .index() .notNullable() .references('id') .inTable('media'); table.unique('release_id'); - })) - .then(() => knex.schema.createTable('releases_photos', (table) => { + }); + + await knex.schema.createTable('releases_photos', (table) => { table.integer('release_id', 16) .notNullable() .references('id') @@ -789,13 +910,31 @@ exports.up = (knex) => Promise.resolve() .onDelete('cascade'); table.text('media_id', 21) + .index() .notNullable() .references('id') .inTable('media'); table.unique(['release_id', 'media_id']); - })) - .then(() => knex.schema.createTable('releases_tags', (table) => { + }); + + await knex.schema.createTable('releases_caps', (table) => { + table.integer('release_id') + .notNullable() + .references('id') + .inTable('releases') + .onDelete('cascade'); + + table.text('media_id') + .index() + .notNullable() + .references('id') + .inTable('media'); + + table.unique(['release_id', 'media_id']); + }); + + await knex.schema.createTable('releases_tags', (table) => { table.integer('tag_id', 12) .references('id') .inTable('tags'); @@ -811,15 +950,21 @@ exports.up = (knex) => Promise.resolve() table.unique(['tag_id', 'release_id']); table.index('tag_id'); table.index('release_id'); - })) - .then(() => knex.schema.createTable('releases_search', (table) => { + + table.enum('source', ['scraper', 'editor', 'implied']) + .notNullable() + .defaultTo('scraper'); + }); + + await knex.schema.createTable('releases_search', (table) => { table.integer('release_id', 16) .references('id') .inTable('releases') .onDelete('cascade'); - })) - .then(() => knex.schema.createTable('movies', (table) => { - table.increments('id', 16); + }); + + await knex.schema.createTable('movies', (table) => { + table.increments('id'); table.integer('entity_id', 12) .references('id') @@ -835,6 +980,7 @@ exports.up = (knex) => Promise.resolve() table.text('url', 1000); table.text('title'); + table.specificType('alt_titles', 'text ARRAY'); table.text('slug'); table.timestamp('date'); @@ -844,6 +990,7 @@ exports.up = (knex) => Promise.resolve() .defaultTo('day'); table.text('description'); + table.integer('photo_count'); table.boolean('deep'); table.text('deep_url', 1000); @@ -862,8 +1009,9 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('movies_scenes', (table) => { + }); + + await knex.schema.createTable('movies_scenes', (table) => { table.integer('movie_id', 16) .notNullable() .references('id') @@ -882,8 +1030,28 @@ exports.up = (knex) => Promise.resolve() .defaultTo(knex.fn.now()); table.index('scene_id'); - })) - .then(() => knex.schema.createTable('movies_covers', (table) => { + }); + + await knex.schema.createTable('movies_tags', (table) => { + table.integer('tag_id') + .references('id') + .inTable('tags'); + + table.integer('movie_id') + .notNullable() + .references('id') + .inTable('movies') + .onDelete('cascade'); + + table.text('original_tag'); + + table.text('source') + .defaultTo('scraper'); + + table.unique(['tag_id', 'movie_id']); + }); + + await knex.schema.createTable('movies_covers', (table) => { table.integer('movie_id', 16) .notNullable() .references('id') @@ -891,13 +1059,15 @@ exports.up = (knex) => Promise.resolve() .onDelete('cascade'); table.text('media_id', 21) + .index() .notNullable() .references('id') .inTable('media'); table.unique(['movie_id', 'media_id']); - })) - .then(() => knex.schema.createTable('movies_trailers', (table) => { + }); + + await knex.schema.createTable('movies_trailers', (table) => { table.integer('movie_id', 16) .unique() .notNullable() @@ -906,11 +1076,13 @@ exports.up = (knex) => Promise.resolve() .onDelete('cascade'); table.text('media_id', 21) + .index() .notNullable() .references('id') .inTable('media'); - })) - .then(() => knex.schema.createTable('movies_posters', (table) => { + }); + + await knex.schema.createTable('movies_teasers', (table) => { table.integer('movie_id', 16) .notNullable() .references('id') @@ -918,14 +1090,32 @@ exports.up = (knex) => Promise.resolve() .onDelete('cascade'); table.text('media_id', 21) + .index() + .notNullable() + .references('id') + .inTable('media'); + + table.unique('movie_id'); + }); + + await knex.schema.createTable('movies_posters', (table) => { + table.integer('movie_id', 16) + .notNullable() + .references('id') + .inTable('movies') + .onDelete('cascade'); + + table.text('media_id', 21) + .index() .notNullable() .references('id') .inTable('media') .onDelete('cascade'); table.unique('movie_id'); - })) - .then(() => knex.schema.createTable('movies_photos', (table) => { + }); + + await knex.schema.createTable('movies_photos', (table) => { table.integer('movie_id', 16) .notNullable() .references('id') @@ -933,20 +1123,23 @@ exports.up = (knex) => Promise.resolve() .onDelete('cascade'); table.text('media_id', 21) + .index() .notNullable() .references('id') .inTable('media'); table.unique(['movie_id', 'media_id']); - })) - .then(() => knex.schema.createTable('movies_search', (table) => { + }); + + await knex.schema.createTable('movies_search', (table) => { table.integer('movie_id', 16) .references('id') .inTable('movies') .onDelete('cascade'); - })) - .then(() => knex.schema.createTable('series', (table) => { - table.increments('id', 16); + }); + + await knex.schema.createTable('series', (table) => { + table.increments('id'); table.integer('entity_id', 12) .references('id') @@ -962,6 +1155,7 @@ exports.up = (knex) => Promise.resolve() table.text('url', 1000); table.text('title'); + table.specificType('alt_titles', 'text ARRAY'); table.text('slug'); table.timestamp('date'); @@ -971,6 +1165,7 @@ exports.up = (knex) => Promise.resolve() .defaultTo('day'); table.text('description'); + table.integer('photo_count'); table.boolean('deep'); table.text('deep_url', 1000); @@ -989,8 +1184,9 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('series_scenes', (table) => { + }); + + await knex.schema.createTable('series_scenes', (table) => { table.integer('serie_id', 16) .notNullable() .references('id') @@ -1007,8 +1203,9 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('series_trailers', (table) => { + }); + + await knex.schema.createTable('series_trailers', (table) => { table.integer('serie_id', 16) .unique() .notNullable() @@ -1017,11 +1214,13 @@ exports.up = (knex) => Promise.resolve() .onDelete('cascade'); table.text('media_id', 21) + .index() .notNullable() .references('id') .inTable('media'); - })) - .then(() => knex.schema.createTable('series_posters', (table) => { + }); + + await knex.schema.createTable('series_teasers', (table) => { table.integer('serie_id', 16) .notNullable() .references('id') @@ -1029,14 +1228,32 @@ exports.up = (knex) => Promise.resolve() .onDelete('cascade'); table.text('media_id', 21) + .index() + .notNullable() + .references('id') + .inTable('media'); + + table.unique('serie_id'); + }); + + await knex.schema.createTable('series_posters', (table) => { + table.integer('serie_id', 16) + .notNullable() + .references('id') + .inTable('series') + .onDelete('cascade'); + + table.text('media_id', 21) + .index() .notNullable() .references('id') .inTable('media') .onDelete('cascade'); table.unique('serie_id'); - })) - .then(() => knex.schema.createTable('series_covers', (table) => { + }); + + await knex.schema.createTable('series_covers', (table) => { table.integer('serie_id', 16) .notNullable() .references('id') @@ -1044,13 +1261,15 @@ exports.up = (knex) => Promise.resolve() .onDelete('cascade'); table.text('media_id', 21) + .index() .notNullable() .references('id') .inTable('media'); table.unique(['serie_id', 'media_id']); - })) - .then(() => knex.schema.createTable('series_photos', (table) => { + }); + + await knex.schema.createTable('series_photos', (table) => { table.integer('serie_id', 16) .notNullable() .references('id') @@ -1058,20 +1277,23 @@ exports.up = (knex) => Promise.resolve() .onDelete('cascade'); table.text('media_id', 21) + .index() .notNullable() .references('id') .inTable('media'); table.unique(['serie_id', 'media_id']); - })) - .then(() => knex.schema.createTable('series_search', (table) => { + }); + + await knex.schema.createTable('series_search', (table) => { table.integer('serie_id', 16) .references('id') .inTable('series') .onDelete('cascade'); - })) - .then(() => knex.schema.createTable('chapters', (table) => { - table.increments('id', 16); + }); + + await knex.schema.createTable('chapters', (table) => { + table.increments('id'); table.integer('release_id', 12) .references('id') @@ -1103,8 +1325,9 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('chapters_posters', (table) => { + }); + + await knex.schema.createTable('chapters_posters', (table) => { table.integer('chapter_id', 16) .notNullable() .references('id') @@ -1112,13 +1335,15 @@ exports.up = (knex) => Promise.resolve() .onDelete('cascade'); table.text('media_id', 21) + .index() .notNullable() .references('id') .inTable('media'); table.unique('chapter_id'); - })) - .then(() => knex.schema.createTable('chapters_photos', (table) => { + }); + + await knex.schema.createTable('chapters_photos', (table) => { table.integer('chapter_id', 16) .notNullable() .references('id') @@ -1126,13 +1351,15 @@ exports.up = (knex) => Promise.resolve() .onDelete('cascade'); table.text('media_id', 21) + .index() .notNullable() .references('id') .inTable('media'); table.unique(['chapter_id', 'media_id']); - })) - .then(() => knex.schema.createTable('chapters_tags', (table) => { + }); + + await knex.schema.createTable('chapters_tags', (table) => { table.integer('tag_id', 12) .references('id') .inTable('tags') @@ -1147,14 +1374,16 @@ exports.up = (knex) => Promise.resolve() table.text('original_tag'); table.unique(['tag_id', 'chapter_id']); - })) - .then(() => knex.schema.createTable('users_roles', (table) => { + }); + + await knex.schema.createTable('users_roles', (table) => { table.string('role') .primary(); table.json('abilities'); - })) - .then(() => knex('users_roles').insert([ + }); + + await knex('users_roles').insert([ { role: 'admin', abilities: JSON.stringify([ // serialization necessary to avoid array being interpreted as a PG array @@ -1176,12 +1405,12 @@ exports.up = (knex) => Promise.resolve() { role: 'user', }, - ])) - .then(() => knex.schema.createTable('users', (table) => { + ]); + + await knex.schema.createTable('users', (table) => { table.increments('id'); table.text('username') - .unique() .notNullable(); table.text('email') @@ -1207,13 +1436,95 @@ exports.up = (knex) => Promise.resolve() .notNullable() .defaultTo(false); + table.specificType('last_ip', 'cidr'); + table.datetime('created_at') .notNullable() .defaultTo(knex.fn.now()); table.datetime('last_login'); - })) - .then(() => knex.schema.createTable('stashes', (table) => { + }); + + await knex.raw(` + CREATE UNIQUE INDEX username_unique_index ON users (LOWER(username)); + CREATE UNIQUE INDEX email_unique_index ON users (LOWER(email)); + `); + + await knex.schema.createTable('users_templates', (table) => { + table.increments('id'); + + table.integer('user_id') + .notNullable() + .references('id') + .inTable('users'); + + table.string('name') + .notNullable(); + + table.text('template') + .notNullable(); + + table.unique(['user_id', 'name']); + + table.datetime('created_at') + .defaultTo(knex.fn.now()); + }); + + await knex.schema.createTable('users_keys', (table) => { + table.increments('id'); + + table.integer('user_id') + .notNullable() + .references('id') + .inTable('users'); + + table.text('key') + .notNullable(); + + table.string('identifier'); + + table.unique(['user_id', 'identifier']); + + table.datetime('last_used_at'); + table.specificType('last_used_ip', 'inet'); + + table.datetime('created_at') + .notNullable() + .defaultTo(knex.fn.now()); + }); + + await knex.schema.createTable('bans', (table) => { + table.increments('id'); + + table.integer('user_id') + .references('id') + .inTable('users') + .onDelete('set null'); + + table.string('username'); + table.specificType('ip', 'cidr'); + + table.boolean('match_all') + .notNullable() + .defaultTo(false); + + table.string('scope'); + table.boolean('shadow'); + + table.integer('banned_by') + .references('id') + .inTable('users') + .onDelete('set null'); + + table.datetime('expires_at') + .notNullable(); + + table.datetime('created_at') + .notNullable() + .defaultTo(knex.fn.now()); + }); + + await knex.schema.createTable('stashes', (table) => { table.increments('id'); table.integer('user_id') @@ -1235,16 +1546,23 @@ exports.up = (knex) => Promise.resolve() .notNullable() .defaultTo(false); + table.json('meta'); + table.text('comment'); + table.datetime('created_at') .notNullable() .defaultTo(knex.fn.now()); table.unique(['user_id', 'slug']); - })) - .then(() => knex.raw(` + }); + + await knex.raw(` CREATE UNIQUE INDEX unique_primary ON stashes (user_id, "primary") WHERE ("primary" = TRUE); - `)) - .then(() => knex.schema.createTable('stashes_scenes', (table) => { + `); + + await knex.schema.createTable('stashes_scenes', (table) => { + table.increments('id'); + table.integer('stash_id') .notNullable() .references('id') @@ -1264,8 +1582,11 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .notNullable() .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('stashes_movies', (table) => { + }); + + await knex.schema.createTable('stashes_movies', (table) => { + table.increments('id'); + table.integer('stash_id') .notNullable() .references('id') @@ -1285,8 +1606,11 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .notNullable() .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('stashes_actors', (table) => { + }); + + await knex.schema.createTable('stashes_actors', (table) => { + table.increments('id'); + table.integer('stash_id') .notNullable() .references('id') @@ -1306,8 +1630,11 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .notNullable() .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('stashes_series', (table) => { + }); + + await knex.schema.createTable('stashes_series', (table) => { + table.increments('id'); + table.integer('stash_id') .notNullable() .references('id') @@ -1327,8 +1654,9 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .notNullable() .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('alerts', (table) => { + }); + + await knex.schema.createTable('alerts', (table) => { table.increments('id'); table.integer('user_id') @@ -1343,11 +1671,38 @@ exports.up = (knex) => Promise.resolve() table.boolean('email') .defaultTo(false); + table.boolean('all') + .defaultTo(true); + + table.boolean('all_actors') + .notNullable() + .defaultTo(true); + + table.boolean('all_entities') + .notNullable() + .defaultTo(true); + + table.boolean('all_tags') + .notNullable() + .defaultTo(true); + + table.boolean('all_matches') + .notNullable() + .defaultTo(true); + + table.boolean('from_preset') + .notNullable() + .defaultTo(false); + + table.json('meta'); + table.text('comment'); + table.datetime('created_at') .notNullable() .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('alerts_scenes', (table) => { + }); + + await knex.schema.createTable('alerts_scenes', (table) => { table.increments('id'); table.integer('alert_id') @@ -1363,8 +1718,9 @@ exports.up = (knex) => Promise.resolve() .onDelete('cascade'); table.unique(['alert_id', 'scene_id']); - })) - .then(() => knex.schema.createTable('alerts_actors', (table) => { + }); + + await knex.schema.createTable('alerts_actors', (table) => { table.increments('id'); table.integer('alert_id') @@ -1380,8 +1736,9 @@ exports.up = (knex) => Promise.resolve() .onDelete('cascade'); table.unique(['alert_id', 'actor_id']); - })) - .then(() => knex.schema.createTable('alerts_tags', (table) => { + }); + + await knex.schema.createTable('alerts_tags', (table) => { table.increments('id'); table.integer('alert_id') @@ -1397,13 +1754,13 @@ exports.up = (knex) => Promise.resolve() .onDelete('cascade'); table.unique(['alert_id', 'tag_id']); - })) - .then(() => knex.schema.createTable('alerts_entities', (table) => { + }); + + await knex.schema.createTable('alerts_entities', (table) => { table.increments('id'); table.integer('alert_id') .notNullable() - .unique() .references('id') .inTable('alerts') .onDelete('cascade'); @@ -1413,8 +1770,9 @@ exports.up = (knex) => Promise.resolve() .references('id') .inTable('entities') .onDelete('cascade'); - })) - .then(() => knex.schema.createTable('alerts_stashes', (table) => { + }); + + await knex.schema.createTable('alerts_stashes', (table) => { table.increments('id'); table.integer('alert_id') @@ -1430,8 +1788,93 @@ exports.up = (knex) => Promise.resolve() .onDelete('cascade'); table.unique(['alert_id', 'stash_id']); - })) - .then(() => knex.schema.createTable('notifications', (table) => { + }); + + await knex.schema.createTable('alerts_matches', (table) => { + table.increments('id'); + + table.integer('alert_id') + .references('id') + .inTable('alerts') + .onDelete('cascade'); + + table.string('property'); + table.string('expression'); + }); + + await knex.schema.createMaterializedView('alerts_users_actors', (view) => { + view.columns('user_id', 'actor_id', 'alert_ids'); + + view.as( + knex('alerts_actors') + .select( + 'alerts.user_id', + 'alerts_actors.actor_id', + knex.raw('array_agg(distinct alerts.id) as alert_ids'), + knex.raw('(alerts_tags.id is null and alerts_entities.id is null and alerts_matches.id is null and related_actors.id is null) as is_only'), + ) + .leftJoin('alerts', 'alerts.id', 'alerts_actors.alert_id') + .leftJoin('alerts_entities', 'alerts_entities.alert_id', 'alerts_actors.alert_id') + .leftJoin('alerts_tags', 'alerts_tags.alert_id', 'alerts_actors.alert_id') + .leftJoin('alerts_matches', 'alerts_matches.alert_id', 'alerts_actors.alert_id') + .leftJoin('alerts_actors as related_actors', (joinBuilder) => { + joinBuilder + .on('related_actors.alert_id', 'alerts_actors.alert_id') + .on('related_actors.actor_id', '!=', 'alerts_actors.actor_id'); + }) + .groupBy(['user_id', 'alerts_actors.actor_id', 'is_only']), + ); + }); + + await knex.schema.createMaterializedView('alerts_users_tags', (view) => { + view.columns('user_id', 'tag_id', 'alert_ids'); + + view.as( + knex('alerts_tags') + .select( + 'alerts.user_id', + 'alerts_tags.tag_id', + knex.raw('array_agg(distinct alerts.id) as alert_ids'), + knex.raw('(alerts_actors.id is null and alerts_entities.id is null and alerts_matches.id is null and related_tags.id is null) as is_only'), + ) + .leftJoin('alerts', 'alerts.id', 'alerts_tags.alert_id') + .leftJoin('alerts_entities', 'alerts_entities.alert_id', 'alerts_tags.alert_id') + .leftJoin('alerts_actors', 'alerts_actors.alert_id', 'alerts_tags.alert_id') + .leftJoin('alerts_matches', 'alerts_matches.alert_id', 'alerts_tags.alert_id') + .leftJoin('alerts_tags as related_tags', (joinBuilder) => { + joinBuilder + .on('related_tags.alert_id', 'alerts_tags.alert_id') + .on('related_tags.tag_id', '!=', 'alerts_tags.tag_id'); + }) + .groupBy(['user_id', 'alerts_tags.tag_id', 'is_only']), + ); + }); + + await knex.schema.createMaterializedView('alerts_users_entities', (view) => { + view.columns('user_id', 'entity_id', 'alert_ids'); + + view.as( + knex('alerts_entities') + .select( + 'alerts.user_id', + 'alerts_entities.entity_id', + knex.raw('array_agg(distinct alerts.id) as alert_ids'), + knex.raw('(alerts_actors.id is null and alerts_tags.id is null and alerts_matches.id is null and related_entities.id is null) as is_only'), + ) + .leftJoin('alerts', 'alerts.id', 'alerts_entities.alert_id') + .leftJoin('alerts_tags', 'alerts_tags.alert_id', 'alerts_entities.alert_id') + .leftJoin('alerts_actors', 'alerts_actors.alert_id', 'alerts_entities.alert_id') + .leftJoin('alerts_matches', 'alerts_matches.alert_id', 'alerts_entities.alert_id') + .leftJoin('alerts_entities as related_entities', (joinBuilder) => { + joinBuilder + .on('related_entities.alert_id', 'alerts_entities.alert_id') + .on('related_entities.entity_id', '!=', 'alerts_entities.entity_id'); + }) + .groupBy(['user_id', 'alerts_entities.entity_id', 'is_only']), + ); + }); + + await knex.schema.createTable('notifications', (table) => { table.increments('id'); table.integer('user_id') @@ -1457,8 +1900,157 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .notNullable() .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('affiliates', (table) => { + }); + + await knex.schema.createTable('fingerprints_types', (table) => { + table.string('type') + .primary(); + }); + + await knex('fingerprints_types').insert([ + 'oshash', + 'phash', + 'md5', + 'blake2', + ].map((type) => ({ type }))); + + await knex.schema.createTable('releases_fingerprints', (table) => { + table.increments('id'); + + table.integer('scene_id') + .notNullable() + .references('id') + .inTable('releases') + .onDelete('cascade'); + + table.string('hash') + .notNullable() + .index(); + + table.string('type') + .notNullable() + .references('type') + .inTable('fingerprints_types'); + + table.integer('duration'); + table.integer('width'); + table.integer('height'); + + table.integer('user_id') + .references('id') + .inTable('users') + .onDelete('set null'); + + table.string('source'); + table.integer('source_submissions'); + table.json('source_meta'); + + table.integer('batch_id') + .notNullable() + .references('id') + .inTable('batches'); + + table.datetime('source_created_at'); + + table.datetime('created_at') + .notNullable() + .defaultTo(knex.fn.now()); + }); + + await knex.raw(` + CREATE UNIQUE INDEX scenes_fingerprints_unique + ON releases_fingerprints (scene_id, hash, source, user_id) + NULLS NOT DISTINCT + `); + + await knex.schema.createTable('scenes_revisions', (table) => { + table.increments('id'); + + table.integer('scene_id') + .references('id') + .inTable('releases') + .onDelete('set null'); + + table.integer('user_id') + .references('id') + .inTable('users') + .onDelete('set null'); + + table.json('base') + .notNullable(); + + table.json('deltas') + .notNullable(); + + table.text('hash') + .notNullable(); + + table.text('comment'); + + table.boolean('approved'); + + table.integer('reviewed_by') + .references('id') + .inTable('users') + .onDelete('set null'); + + table.datetime('reviewed_at'); + table.text('feedback'); + + table.datetime('applied_at'); + + table.datetime('created_at') + .notNullable() + .defaultTo(knex.fn.now()); + }); + + await knex.schema.createTable('actors_revisions', (table) => { + table.increments('id'); + + table.integer('actor_id') + .references('id') + .inTable('actors') + .onDelete('set null'); + + table.integer('profile_id') + .references('id') + .inTable('actors_profiles') + .onDelete('set null'); + + table.integer('user_id') + .references('id') + .inTable('users') + .onDelete('set null'); + + table.json('base') + .notNullable(); + + table.json('deltas') + .notNullable(); + + table.text('hash') + .notNullable(); + + table.text('comment'); + + table.boolean('approved'); + + table.integer('reviewed_by') + .references('id') + .inTable('users') + .onDelete('set null'); + + table.datetime('reviewed_at'); + table.text('feedback'); + + table.datetime('applied_at'); + + table.datetime('created_at') + .notNullable() + .defaultTo(knex.fn.now()); + }); + + await knex.schema.createTable('affiliates', (table) => { table.string('id') .primary() .unique() @@ -1469,18 +2061,18 @@ exports.up = (knex) => Promise.resolve() .inTable('entities'); table.text('url'); - table.text('parameters'); + table.json('parameters'); table.unique(['entity_id', 'url']); - table.unique(['entity_id', 'parameters']); table.text('comment'); table.datetime('created_at') .notNullable() .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('banners', (table) => { + }); + + await knex.schema.createTable('banners', (table) => { table.string('id') .primary() .unique() @@ -1504,11 +2096,13 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .notNullable() .defaultTo(knex.fn.now()); - })) - .then(() => knex.raw(` + }); + + await knex.raw(` ALTER TABLE banners ADD COLUMN ratio numeric GENERATED ALWAYS AS (ROUND(width::decimal/ height::decimal, 2)) STORED; - `)) - .then(() => knex.schema.createTable('banners_tags', (table) => { + `); + + await knex.schema.createTable('banners_tags', (table) => { table.increments('id'); table.string('banner_id') @@ -1526,8 +2120,9 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .notNullable() .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('campaigns', (table) => { + }); + + await knex.schema.createTable('campaigns', (table) => { table.increments('id'); table.integer('entity_id', 12) @@ -1550,8 +2145,9 @@ exports.up = (knex) => Promise.resolve() table.datetime('created_at') .notNullable() .defaultTo(knex.fn.now()); - })) - .then(() => knex.schema.createTable('random_campaign', (table) => { + }); + + await knex.schema.createTable('random_campaign', (table) => { table.integer('id') .notNullable() .references('id') @@ -1574,522 +2170,159 @@ exports.up = (knex) => Promise.resolve() table.integer('parent_id') .references('id') .inTable('entities'); - })) - // SEARCH AND SORT - .then(() => { // eslint-disable-line arrow-body-style - // allow vim fold - return knex.raw(` - ALTER TABLE releases_search ADD COLUMN document tsvector; - ALTER TABLE movies_search ADD COLUMN document tsvector; - ALTER TABLE series_search ADD COLUMN document tsvector; - - /* allow scenes without dates to be mixed inbetween scenes with dates */ - ALTER TABLE releases - ADD COLUMN effective_date timestamptz - GENERATED ALWAYS AS (COALESCE(date, created_at)) STORED; - - ALTER TABLE movies - ADD COLUMN effective_date timestamptz - GENERATED ALWAYS AS (COALESCE(date, created_at)) STORED; - `); - }) - // INDEXES - .then(() => { // eslint-disable-line arrow-body-style - // allow vim fold - return knex.raw(` - CREATE UNIQUE INDEX unique_actor_slugs_network ON actors (slug, entity_id, entry_id); - CREATE UNIQUE INDEX unique_actor_slugs ON actors (slug) WHERE entity_id IS NULL; - - CREATE UNIQUE INDEX unique_entity_campaigns_banner_url ON campaigns (entity_id, url, banner_id) WHERE affiliate_id IS NULL; - CREATE UNIQUE INDEX unique_entity_campaigns_url ON campaigns (entity_id, url) WHERE banner_id IS NULL AND affiliate_id IS NULL; - CREATE UNIQUE INDEX unique_entity_campaigns_banner_affiliate ON campaigns (entity_id, affiliate_id, banner_id) WHERE url IS NULL; - CREATE UNIQUE INDEX unique_entity_campaigns_affiliate ON campaigns (entity_id, affiliate_id) WHERE banner_id IS NULL AND url IS NULL; - - CREATE UNIQUE INDEX releases_search_unique ON releases_search (release_id); - CREATE UNIQUE INDEX movies_search_unique ON movies_search (movie_id); - CREATE INDEX releases_search_index ON releases_search USING GIN (document); - CREATE INDEX movies_search_index ON movies_search USING GIN (document); - CREATE UNIQUE INDEX series_search_unique ON series_search (serie_id); - CREATE INDEX series_search_index ON series_search USING GIN (document); - `); - }) - // FUNCTIONS - .then(() => { // eslint-disable-line arrow-body-style - // allow vim fold - return knex.raw(` - CREATE EXTENSION IF NOT EXISTS pg_trgm; - - CREATE FUNCTION current_user_id() RETURNS INTEGER AS $$ - /* if the user ID is undefined, the adapter will pass it as a string, which cannot be cast as NULL by ::integer */ - SELECT NULLIF(current_setting('user.id', true), '')::integer; - $$ LANGUAGE SQL STABLE; - - /* We need both the release entries and their search ranking, and PostGraphile does not seem to allow virtual foreign keys on function results. - * Using a table as a proxy for the search results allows us to get both a reference to the releases table, and the ranking. - * A composite type does not seem to be compatible with PostGraphile's @sortable, and a view does not allow for many native constraints */ - CREATE TABLE releases_search_results (release_id integer, rank real, FOREIGN KEY (release_id) REFERENCES releases (id)); - CREATE TABLE movies_search_results (movie_id integer, rank real, FOREIGN KEY (movie_id) REFERENCES movies (id)); - - CREATE FUNCTION search_releases(query text) RETURNS SETOF releases_search_results AS $$ - SELECT results.release_id, ts_rank(results.document::tsvector, curate_search_query(query)) as rank - FROM ( - SELECT releases_search.release_id, document - FROM releases_search - WHERE document::tsvector @@ curate_search_query(query) - ) AS results - ORDER BY rank DESC; - $$ LANGUAGE SQL STABLE; - - CREATE FUNCTION search_movies(query text) RETURNS SETOF movies_search_results AS $$ - SELECT movies.id, ranks.rank FROM ( - SELECT - movies_search.movie_id, - ts_rank(movies_search.document, to_tsquery('english', array_to_string(array(SELECT * FROM regexp_matches(query, '[A-Za-zÀ-ÖØ-öø-ÿ0-9]+', 'g')), '|'))) AS rank - FROM movies_search - ) ranks - LEFT JOIN movies ON movies.id = ranks.movie_id - WHERE ranks.rank > 0 - ORDER BY ranks.rank DESC; - $$ LANGUAGE SQL STABLE; - - CREATE FUNCTION search_entities(search text) RETURNS SETOF entities AS $$ - SELECT * FROM entities - WHERE - name ILIKE ('%' || TRIM(search) || '%') OR - slug ILIKE ('%' || TRIM(search) || '%') OR - array_to_string(alias, '') ILIKE ('%' || TRIM(search) || '%') OR - replace(array_to_string(alias, ''), ' ', '') ILIKE ('%' || TRIM(search) || '%') OR - url ILIKE ('%' || search || '%') - ORDER BY similarity(entities.name, search) DESC, name ASC - $$ LANGUAGE SQL STABLE; - - CREATE FUNCTION search_actors(query text, min_length smallint DEFAULT 2) RETURNS SETOF actors AS $$ - SELECT * FROM actors - WHERE query IS NULL - OR length(query) >= min_length - AND CASE - WHEN length(query) > 1 THEN name ILIKE ('%' || TRIM(query) || '%') - WHEN length(query) = 1 THEN name ILIKE (TRIM(query) || '%') - ELSE true - END - $$ LANGUAGE SQL STABLE; - - CREATE FUNCTION search_tags(query text, min_length smallint DEFAULT 2) RETURNS SETOF tags AS $$ - SELECT aliases.* FROM tags - LEFT JOIN tags AS aliases ON aliases.id = tags.alias_for OR (tags.alias_for IS NULL AND aliases.id = tags.id) - WHERE length(query) >= min_length - AND tags.name ILIKE ('%' || TRIM(query) || '%') - GROUP BY aliases.id - ORDER BY similarity(aliases.slug, query) DESC, slug ASC - $$ LANGUAGE SQL STABLE; - - CREATE FUNCTION actors_tags(actor actors, selectable_tags text[]) RETURNS SETOF tags AS $$ - SELECT tags.* - FROM releases_actors - LEFT JOIN - releases_tags ON releases_tags.release_id = releases_actors.release_id - LEFT JOIN - tags ON tags.id = releases_tags.tag_id - WHERE - releases_actors.actor_id = actor.id - AND - CASE WHEN array_length(selectable_tags, 1) IS NOT NULL - THEN tags.slug = ANY(selectable_tags) - ELSE true - END - GROUP BY tags.id - ORDER BY tags.name; - $$ LANGUAGE SQL STABLE; - - CREATE FUNCTION actors_channels(actor actors) RETURNS SETOF entities AS $$ - SELECT entities.* - FROM releases_actors - LEFT JOIN releases ON releases.id = releases_actors.release_id - LEFT JOIN entities ON entities.id = releases.entity_id - WHERE releases_actors.actor_id = actor.id - GROUP BY entities.id; - $$ LANGUAGE SQL STABLE; - - CREATE FUNCTION actors_actors(actor actors) RETURNS SETOF actors AS $$ - SELECT actors.* - FROM releases_actors - LEFT JOIN releases_actors AS associated_actors ON associated_actors.release_id = releases_actors.release_id - LEFT JOIN actors ON actors.id = associated_actors.actor_id - WHERE releases_actors.actor_id = actor.id - AND NOT actors.id = actor.id - GROUP BY actors.id - ORDER BY actors.name; - $$ LANGUAGE SQL STABLE; - - CREATE FUNCTION entities_scenes(entity entities) RETURNS SETOF releases AS $$ - WITH RECURSIVE children AS ( - SELECT entities.id - FROM entities - WHERE entities.id = entity.id - - UNION ALL - - SELECT entities.id - FROM entities - INNER JOIN children ON children.id = entities.parent_id - ) - - SELECT releases FROM releases - INNER JOIN children ON children.id = releases.entity_id - - UNION - - SELECT releases FROM releases - INNER JOIN children ON children.id = releases.studio_id; - $$ LANGUAGE SQL STABLE; - - CREATE OR REPLACE FUNCTION entities_scene_total(entity entities) RETURNS bigint AS $$ - SELECT COUNT(id) - FROM releases - WHERE releases.entity_id = entity.id; - $$ LANGUAGE SQL STABLE; - - CREATE FUNCTION entities_scene_tags(entity entities, selectable_tags text[]) RETURNS SETOF tags AS $$ - SELECT tags.* - FROM releases - LEFT JOIN - releases_tags ON releases_tags.release_id = releases.id - LEFT JOIN - tags ON tags.id = releases_tags.tag_id - WHERE - releases.entity_id = entity.id - AND - CASE WHEN array_length(selectable_tags, 1) IS NOT NULL - THEN tags.slug = ANY(selectable_tags) - ELSE true - END - GROUP BY tags.id - ORDER BY tags.name; - $$ LANGUAGE SQL STABLE; - - /* GraphQL/Postgraphile 'every' applies to the data, will only include scenes for which every assigned tag is selected, - instead of what we want; scenes with every selected tag, but possibly also some others */ - CREATE FUNCTION actors_scenes(actor actors, selected_tags text[], mode text DEFAULT 'all') RETURNS SETOF releases AS $$ - SELECT releases.* - FROM releases - LEFT JOIN - releases_actors ON releases_actors.release_id = releases.id - LEFT JOIN - releases_tags ON releases_tags.release_id = releases.id - LEFT JOIN - tags ON tags.id = releases_tags.tag_id - WHERE releases_actors.actor_id = actor.id - AND CASE - /* match at least one of the selected tags */ - WHEN mode = 'any' - AND array_length(selected_tags, 1) > 0 - THEN tags.slug = ANY(selected_tags) - ELSE true - END - GROUP BY releases.id - HAVING CASE - /* match all of the selected tags */ - WHEN mode = 'all' - AND array_length(selected_tags, 1) > 0 - THEN COUNT( - CASE WHEN tags.slug = ANY(selected_tags) - THEN true - END - ) = array_length(selected_tags, 1) - ELSE true - END; - $$ LANGUAGE SQL STABLE; - - CREATE FUNCTION tags_scenes(tag tags, selected_tags text[], mode text DEFAULT 'all') RETURNS SETOF releases AS $$ - SELECT releases.* - FROM releases - LEFT JOIN - releases_actors ON releases_actors.release_id = releases.id - LEFT JOIN - releases_tags ON releases_tags.release_id = releases.id - LEFT JOIN - tags ON tags.id = releases_tags.tag_id - WHERE releases_tags.tag_id = tag.id - GROUP BY releases.id; - $$ LANGUAGE SQL STABLE; - - CREATE FUNCTION movies_actors(movie movies) RETURNS SETOF actors AS $$ - SELECT actors.* - FROM movies_scenes - LEFT JOIN - releases ON releases.id = movies_scenes.scene_id - LEFT JOIN - releases_actors ON releases_actors.release_id = releases.id - LEFT JOIN - actors ON actors.id = releases_actors.actor_id - WHERE movies_scenes.movie_id = movie.id - AND actors.id IS NOT NULL - GROUP BY actors.id - ORDER BY actors.name, actors.gender - $$ LANGUAGE SQL STABLE; - - CREATE FUNCTION movies_tags(movie movies) RETURNS SETOF tags AS $$ - SELECT tags.* - FROM movies_scenes - LEFT JOIN - releases ON releases.id = movies_scenes.scene_id - LEFT JOIN - releases_tags ON releases_tags.release_id = releases.id - LEFT JOIN - tags ON tags.id = releases_tags.tag_id - WHERE movies_scenes.movie_id = movie.id - AND tags.id IS NOT NULL - GROUP BY tags.id - ORDER BY tags.priority DESC - $$ LANGUAGE SQL STABLE; - - CREATE FUNCTION movies_scenes_photos(movie movies) RETURNS SETOF media AS $$ - SELECT media.* - FROM movies_scenes - LEFT JOIN - releases ON releases.id = movies_scenes.scene_id - INNER JOIN - releases_photos ON releases_photos.release_id = releases.id - LEFT JOIN - media ON media.id = releases_photos.media_id - WHERE movies_scenes.movie_id = movie.id - GROUP BY media.id - ORDER BY media.index ASC - $$ LANGUAGE SQL STABLE; - - CREATE FUNCTION releases_is_new(release releases) RETURNS boolean AS $$ - SELECT EXISTS(SELECT true WHERE (SELECT id FROM batches ORDER BY created_at DESC LIMIT 1) = release.created_batch_id); - $$ LANGUAGE sql STABLE; - - CREATE FUNCTION series_actors(serie series) RETURNS SETOF actors AS $$ - SELECT actors.* - FROM series_scenes - LEFT JOIN - releases ON releases.id = series_scenes.scene_id - LEFT JOIN - releases_actors ON releases_actors.release_id = releases.id - LEFT JOIN - actors ON actors.id = releases_actors.actor_id - WHERE series_scenes.serie_id = serie.id - AND actors.id IS NOT NULL - GROUP BY actors.id - ORDER BY actors.name, actors.gender - $$ LANGUAGE SQL STABLE; - - CREATE FUNCTION series_tags(serie series) RETURNS SETOF tags AS $$ - SELECT tags.* - FROM series_scenes - LEFT JOIN - releases ON releases.id = series_scenes.scene_id - LEFT JOIN - releases_tags ON releases_tags.release_id = releases.id - LEFT JOIN - tags ON tags.id = releases_tags.tag_id - WHERE series_scenes.serie_id = serie.id - AND tags.id IS NOT NULL - GROUP BY tags.id - ORDER BY tags.priority DESC - $$ LANGUAGE SQL STABLE; - - CREATE FUNCTION series_scenes_photos(serie series) RETURNS SETOF media AS $$ - SELECT media.* - FROM series_scenes - LEFT JOIN - releases ON releases.id = series_scenes.scene_id - INNER JOIN - releases_photos ON releases_photos.release_id = releases.id - LEFT JOIN - media ON media.id = releases_photos.media_id - WHERE series_scenes.serie_id = serie.id - GROUP BY media.id - ORDER BY media.index ASC - $$ LANGUAGE SQL STABLE; - - CREATE FUNCTION get_random_campaign(min_ratio decimal default 0, max_ratio decimal default 1000.0) RETURNS random_campaign AS $$ - SELECT * FROM ( - SELECT DISTINCT ON (CASE WHEN parent_id IS NOT NULL THEN parent_id ELSE entity_id END) - id, banner_id, url, entity_id, affiliate_id, parent_id - FROM ( - SELECT - campaigns.*, entities.parent_id as parent_id - FROM campaigns - LEFT JOIN entities ON entities.id = campaigns.entity_id - LEFT JOIN banners ON banners.id = campaigns.banner_id - WHERE banner_id IS NOT NULL - AND ratio >= min_ratio - AND ratio <= max_ratio - ORDER BY RANDOM() - ) random_campaigns - ) random_banners - ORDER BY RANDOM() - LIMIT 1; - $$ LANGUAGE SQL STABLE; - `); - }) - // VIEWS AND COMMENTS - .then(() => { // eslint-disable-line arrow-body-style - // allow vim fold - return knex.raw(` - CREATE MATERIALIZED VIEW releases_not_showcased AS ( - SELECT releases.id AS release_id FROM releases - LEFT JOIN entities AS channels ON channels.id = releases.entity_id - LEFT JOIN entities AS studios ON studios.id = releases.studio_id - LEFT JOIN entities AS networks ON networks.id = channels.parent_id - WHERE (studios.showcased = false) - OR (channels.showcased = false AND studios.showcased IS NOT true) - OR (networks.showcased = false AND channels.showcased IS NOT true AND studios.showcased IS NOT true) - ); - - CREATE UNIQUE INDEX ON releases_not_showcased (release_id); - COMMENT ON MATERIALIZED VIEW releases_not_showcased IS E'@foreignKey (release_id) references releases (id)'; - - COMMENT ON COLUMN users.password IS E'@omit'; - COMMENT ON COLUMN users.email IS E'@omit'; - COMMENT ON COLUMN users.email_verified IS E'@omit'; - COMMENT ON COLUMN users.abilities IS E'@omit'; - - COMMENT ON COLUMN actors.height IS E'@omit read,update,create,delete,all,many'; - COMMENT ON COLUMN actors.weight IS E'@omit read,update,create,delete,all,many'; - COMMENT ON COLUMN actors.penis_length IS E'@omit read,update,create,delete,all,many'; - COMMENT ON COLUMN actors.penis_girth IS E'@omit read,update,create,delete,all,many'; - - COMMENT ON FUNCTION entities_scenes IS E'@sortable'; - - COMMENT ON FUNCTION actors_tags IS E'@sortable'; - COMMENT ON FUNCTION actors_channels IS E'@sortable'; - COMMENT ON FUNCTION actors_actors IS E'@sortable'; - COMMENT ON FUNCTION actors_scenes IS E'@sortable'; - COMMENT ON FUNCTION tags_scenes IS E'@sortable'; - - COMMENT ON FUNCTION search_releases IS E'@sortable'; - COMMENT ON FUNCTION search_entities IS E'@sortable'; - COMMENT ON FUNCTION search_actors IS E'@sortable'; - COMMENT ON FUNCTION search_movies IS E'@sortable'; - COMMENT ON FUNCTION search_tags IS E'@sortable'; - `); - }) - // POLICIES - .then(() => { // eslint-disable-line arrow-body-style - // allow vim fold - return knex.raw(` - GRANT ALL ON ALL TABLES IN SCHEMA public TO :visitor; - GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO :visitor; - - REVOKE ALL ON users FROM :visitor; - GRANT SELECT (id, username, role, identity_verified, created_at) ON users TO :visitor; - - ALTER TABLE stashes ENABLE ROW LEVEL SECURITY; - ALTER TABLE stashes_scenes ENABLE ROW LEVEL SECURITY; - ALTER TABLE stashes_movies ENABLE ROW LEVEL SECURITY; - ALTER TABLE stashes_actors ENABLE ROW LEVEL SECURITY; - ALTER TABLE stashes_series ENABLE ROW LEVEL SECURITY; - - CREATE POLICY stashes_policy_select ON stashes FOR SELECT USING (stashes.public OR stashes.user_id = current_user_id()); - CREATE POLICY stashes_policy_update ON stashes FOR UPDATE USING (stashes.public OR stashes.user_id = current_user_id()); - CREATE POLICY stashes_policy_delete ON stashes FOR DELETE USING (stashes.public OR stashes.user_id = current_user_id()); - CREATE POLICY stashes_policy_insert ON stashes FOR INSERT WITH CHECK (true); - - CREATE POLICY stashes_policy ON stashes_scenes - USING (EXISTS ( - SELECT * - FROM stashes - WHERE stashes.id = stashes_scenes.stash_id - AND (stashes.user_id = current_user_id() OR stashes.public) - )); - - CREATE POLICY stashes_policy ON stashes_movies - USING (EXISTS ( - SELECT * - FROM stashes - WHERE stashes.id = stashes_movies.stash_id - AND (stashes.user_id = current_user_id() OR stashes.public) - )); - - CREATE POLICY stashes_policy ON stashes_actors - USING (EXISTS ( - SELECT * - FROM stashes - WHERE stashes.id = stashes_actors.stash_id - AND (stashes.user_id = current_user_id() OR stashes.public) - )); - - CREATE POLICY stashes_policy ON stashes_series - USING (EXISTS ( - SELECT * - FROM stashes - WHERE stashes.id = stashes_series.stash_id - AND (stashes.user_id = current_user_id() OR stashes.public) - )); - - ALTER TABLE alerts ENABLE ROW LEVEL SECURITY; - ALTER TABLE alerts_tags ENABLE ROW LEVEL SECURITY; - ALTER TABLE alerts_scenes ENABLE ROW LEVEL SECURITY; - ALTER TABLE alerts_actors ENABLE ROW LEVEL SECURITY; - ALTER TABLE alerts_entities ENABLE ROW LEVEL SECURITY; - ALTER TABLE alerts_stashes ENABLE ROW LEVEL SECURITY; - - CREATE POLICY alerts_policy_select ON alerts FOR SELECT USING (alerts.user_id = current_user_id()); - CREATE POLICY alerts_policy_update ON alerts FOR UPDATE USING (alerts.user_id = current_user_id()); - CREATE POLICY alerts_policy_delete ON alerts FOR DELETE USING (alerts.user_id = current_user_id()); - CREATE POLICY alerts_policy_insert ON alerts FOR INSERT WITH CHECK (true); - - CREATE POLICY alerts_policy ON alerts_scenes - USING (EXISTS ( - SELECT * - FROM alerts - WHERE alerts.id = alerts_scenes.alert_id - AND alerts.user_id = current_user_id() - )); - - CREATE POLICY alerts_policy ON alerts_actors - USING (EXISTS ( - SELECT * - FROM alerts - WHERE alerts.id = alerts_actors.alert_id - AND alerts.user_id = current_user_id() - )); - - CREATE POLICY alerts_policy ON alerts_entities - USING (EXISTS ( - SELECT * - FROM alerts - WHERE alerts.id = alerts_entities.alert_id - AND alerts.user_id = current_user_id() - )); - - CREATE POLICY alerts_policy ON alerts_tags - USING (EXISTS ( - SELECT * - FROM alerts - WHERE alerts.id = alerts_tags.alert_id - AND alerts.user_id = current_user_id() - )); - - CREATE POLICY alerts_policy ON alerts_stashes - USING (EXISTS ( - SELECT * - FROM alerts - WHERE alerts.id = alerts_stashes.alert_id - AND alerts.user_id = current_user_id() - )); - - ALTER TABLE notifications ENABLE ROW LEVEL SECURITY; - - CREATE POLICY notifications_policy_select ON notifications FOR SELECT USING (notifications.user_id = current_user_id()); - CREATE POLICY notifications_policy_update ON notifications FOR UPDATE USING (notifications.user_id = current_user_id()); - CREATE POLICY notifications_policy_delete ON notifications FOR DELETE USING (notifications.user_id = current_user_id()); - CREATE POLICY notifications_policy_insert ON notifications FOR INSERT WITH CHECK (true); - - ALTER TABLE releases_photos ENABLE ROW LEVEL SECURITY; - CREATE POLICY releases_photos_select ON releases_photos FOR SELECT USING (current_user_id() IS NOT NULL); - - ALTER TABLE releases_trailers ENABLE ROW LEVEL SECURITY; - CREATE POLICY releases_trailers_select ON releases_trailers FOR SELECT USING (current_user_id() IS NOT NULL); - `, { - visitor: knex.raw(config.database.query.user), - }); }); + // SEARCH AND SORT + await knex.raw(` + ALTER TABLE releases_search ADD COLUMN document tsvector; + ALTER TABLE movies_search ADD COLUMN document tsvector; + ALTER TABLE series_search ADD COLUMN document tsvector; + + /* allow scenes without dates to be mixed inbetween scenes with dates */ + ALTER TABLE releases + ADD COLUMN effective_date timestamptz + GENERATED ALWAYS AS (COALESCE(date, created_at)) STORED; + + ALTER TABLE movies + ADD COLUMN effective_date timestamptz + GENERATED ALWAYS AS (COALESCE(date, created_at)) STORED; + + ALTER TABLE series + ADD COLUMN effective_date timestamptz + GENERATED ALWAYS AS (COALESCE(date, created_at)) STORED; + `); + + // INDEXES + await knex.raw(` + CREATE UNIQUE INDEX unique_actor_slugs_network ON actors (slug, entity_id, entry_id); + CREATE UNIQUE INDEX unique_actor_slugs ON actors (slug) WHERE entity_id IS NULL; + + CREATE UNIQUE INDEX unique_entity_campaigns_banner_url ON campaigns (entity_id, url, banner_id) WHERE affiliate_id IS NULL; + CREATE UNIQUE INDEX unique_entity_campaigns_url ON campaigns (entity_id, url) WHERE banner_id IS NULL AND affiliate_id IS NULL; + CREATE UNIQUE INDEX unique_entity_campaigns_banner_affiliate ON campaigns (entity_id, affiliate_id, banner_id) WHERE url IS NULL; + CREATE UNIQUE INDEX unique_entity_campaigns_affiliate ON campaigns (entity_id, affiliate_id) WHERE banner_id IS NULL AND url IS NULL; + + CREATE UNIQUE INDEX releases_search_unique ON releases_search (release_id); + CREATE UNIQUE INDEX movies_search_unique ON movies_search (movie_id); + CREATE INDEX releases_search_index ON releases_search USING GIN (document); + CREATE INDEX movies_search_index ON movies_search USING GIN (document); + CREATE UNIQUE INDEX series_search_unique ON series_search (serie_id); + CREATE INDEX series_search_index ON series_search USING GIN (document); + `); + + // FUNCTIONS + await knex.raw(` + CREATE EXTENSION IF NOT EXISTS pg_trgm; + + CREATE FUNCTION current_user_id() RETURNS INTEGER AS $$ + /* if the user ID is undefined, the adapter will pass it as a string, which cannot be cast as NULL by ::integer */ + SELECT NULLIF(current_setting('user.id', true), '')::integer; + $$ LANGUAGE SQL STABLE; + `); + + // VIEWS AND COMMENTS + await knex.raw(` + CREATE MATERIALIZED VIEW actors_meta AS ( + SELECT + actors.id as actor_id, + COUNT(DISTINCT stashes_actors)::integer as stashed, + COUNT(DISTINCT releases_actors)::integer as scenes, + row_to_json(avatars) as avatar + FROM actors + LEFT JOIN stashes_actors ON stashes_actors.actor_id = actors.id + LEFT JOIN releases_actors ON releases_actors.actor_id = actors.id + LEFT JOIN media AS avatars ON avatars.id = actors.avatar_media_id + GROUP BY + actors.id, + avatars.id + ); + + CREATE MATERIALIZED VIEW scenes_meta AS ( + SELECT + releases.id as scene_id, + COUNT(DISTINCT stashes_scenes)::integer as stashed + FROM releases + LEFT JOIN stashes_scenes ON stashes_scenes.scene_id = releases.id + GROUP BY releases.id + ); + + CREATE MATERIALIZED VIEW movies_meta AS ( + SELECT + movie_id, + stashed, + stashed_scenes, + stashed + stashed_scenes as stashed_total + FROM ( + SELECT + movies.id as movie_id, + COUNT(DISTINCT stashes_movies)::integer as stashed, + COUNT(DISTINCT stashes_scenes)::integer as stashed_scenes + FROM movies + LEFT JOIN stashes_movies ON stashes_movies.movie_id = movies.id + LEFT JOIN movies_scenes ON movies_scenes.movie_id = movies.id + LEFT JOIN stashes_scenes ON stashes_scenes.scene_id = movies_scenes.scene_id + GROUP BY movies.id + ) AS meta + ); + + CREATE MATERIALIZED VIEW stashes_meta AS ( + SELECT + stashes.id as stash_id, + COUNT(DISTINCT stashes_scenes)::integer as stashed_scenes, + COUNT(DISTINCT stashes_movies)::integer as stashed_movies, + COUNT(DISTINCT stashes_actors)::integer as stashed_actors + FROM stashes + LEFT JOIN stashes_scenes ON stashes_scenes.stash_id = stashes.id + LEFT JOIN stashes_movies ON stashes_movies.stash_id = stashes.id + LEFT JOIN stashes_actors ON stashes_actors.stash_id = stashes.id + GROUP BY stashes.id + ); + + CREATE MATERIALIZED VIEW releases_not_showcased AS ( + SELECT releases.id AS release_id FROM releases + LEFT JOIN entities AS channels ON channels.id = releases.entity_id + LEFT JOIN entities AS studios ON studios.id = releases.studio_id + LEFT JOIN entities AS networks ON networks.id = channels.parent_id + WHERE (studios.showcased = false) + OR (channels.showcased = false AND studios.showcased IS NOT true) + OR (networks.showcased = false AND channels.showcased IS NOT true AND studios.showcased IS NOT true) + ); + + CREATE UNIQUE INDEX ON releases_not_showcased (release_id); + + CREATE MATERIALIZED VIEW releases_summaries AS ( + SELECT + releases.id as release_id, + channels.slug as channel_slug, + channels.type as channel_type, + networks.slug as network_slug, + networks.type as network_type, + parent_networks.slug as parent_network_slug, + parent_networks.type as parent_network_type, + studios.showcased IS NOT false + AND (channels.showcased IS NOT false OR COALESCE(studios.showcased, false) = true) + AND (networks.showcased IS NOT false OR COALESCE(channels.showcased, false) = true OR COALESCE(studios.showcased, false) = true) + AS showcased, + batches.showcased AS batch_showcased, + releases.effective_date, + releases.created_at, + array_agg(tags.slug ORDER BY tags.priority DESC) FILTER (WHERE tags.slug IS NOT NULL) AS tags + FROM releases + LEFT JOIN releases_tags ON releases_tags.release_id = releases.id + LEFT JOIN tags ON tags.id = releases_tags.tag_id + LEFT JOIN entities AS channels ON channels.id = releases.entity_id + LEFT JOIN entities AS studios ON studios.id = releases.studio_id + LEFT JOIN entities AS networks ON networks.id = channels.parent_id + LEFT JOIN entities AS parent_networks ON parent_networks.id = networks.parent_id + LEFT JOIN batches ON batches.id = releases.updated_batch_id + GROUP BY releases.id, studios.showcased, batches.showcased, + channels.showcased, channels.slug, channels.type, + networks.showcased, networks.slug, networks.type, + parent_networks.slug, parent_networks.type + ); + `); +}; + exports.down = (knex) => { // eslint-disable-line arrow-body-style // allow vim fold return knex.raw(` @@ -2099,25 +2332,33 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style DROP TABLE IF EXISTS releases_posters CASCADE; DROP TABLE IF EXISTS releases_photos CASCADE; DROP TABLE IF EXISTS releases_covers CASCADE; + DROP TABLE IF EXISTS releases_caps CASCADE; DROP TABLE IF EXISTS releases_trailers CASCADE; DROP TABLE IF EXISTS releases_teasers CASCADE; DROP TABLE IF EXISTS releases_tags CASCADE; + DROP TABLE IF EXISTS releases_fingerprints CASCADE; DROP TABLE IF EXISTS releases_search CASCADE; DROP TABLE IF EXISTS movies_search CASCADE; + DROP TABLE IF EXISTS scenes_revisions CASCADE; + DROP TABLE IF EXISTS movies_scenes CASCADE; DROP TABLE IF EXISTS movies_covers CASCADE; DROP TABLE IF EXISTS movies_posters CASCADE; DROP TABLE IF EXISTS movies_photos CASCADE; DROP TABLE IF EXISTS movies_trailers CASCADE; + DROP TABLE IF EXISTS movies_teasers CASCADE; + DROP TABLE IF EXISTS movies_tags CASCADE; DROP TABLE IF EXISTS stashes_series CASCADE; DROP TABLE IF EXISTS series_scenes CASCADE; DROP TABLE IF EXISTS series_trailers CASCADE; + DROP TABLE IF EXISTS series_teasers CASCADE; DROP TABLE IF EXISTS series_posters CASCADE; DROP TABLE IF EXISTS series_covers CASCADE; DROP TABLE IF EXISTS series_photos CASCADE; DROP TABLE IF EXISTS series_search CASCADE; + DROP TABLE IF EXISTS series_revisions CASCADE; DROP TABLE IF EXISTS clips_tags CASCADE; DROP TABLE IF EXISTS clips_posters CASCADE; @@ -2134,6 +2375,7 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style DROP TABLE IF EXISTS affiliates CASCADE; DROP TABLE IF EXISTS batches CASCADE; + DROP TABLE IF EXISTS fingerprints_types CASCADE; DROP TABLE IF EXISTS actors_avatars CASCADE; DROP TABLE IF EXISTS actors_photos CASCADE; @@ -2141,6 +2383,8 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style DROP TABLE IF EXISTS actors_profiles CASCADE; DROP TABLE IF EXISTS actors_tattoos CASCADE; DROP TABLE IF EXISTS actors_piercings CASCADE; + DROP TABLE IF EXISTS actors_revisions CASCADE; + DROP TABLE IF EXISTS actors_socials CASCADE; DROP TABLE IF EXISTS body CASCADE; DROP TABLE IF EXISTS entities_tags CASCADE; @@ -2182,8 +2426,12 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style DROP TABLE IF EXISTS alerts_tags CASCADE; DROP TABLE IF EXISTS alerts_entities CASCADE; DROP TABLE IF EXISTS alerts_stashes CASCADE; + DROP TABLE IF EXISTS alerts_matches CASCADE; DROP TABLE IF EXISTS alerts CASCADE; + DROP TABLE IF EXISTS bans CASCADE; + DROP TABLE IF EXISTS users_keys CASCADE; + DROP TABLE IF EXISTS users_templates CASCADE; DROP TABLE IF EXISTS users CASCADE; DROP TABLE IF EXISTS users_roles CASCADE; diff --git a/migrations/20230703013413_random_campaign.js b/migrations/20230703013413_random_campaign.js deleted file mode 100644 index 5911a30f..00000000 --- a/migrations/20230703013413_random_campaign.js +++ /dev/null @@ -1,58 +0,0 @@ -exports.up = async (knex) => { - await knex.schema.alterTable('random_campaign', (table) => { - table.integer('id') - .notNullable() - .references('id') - .inTable('campaigns'); - }); - - await knex.raw(` - CREATE OR REPLACE FUNCTION get_random_campaign(min_ratio decimal default 0, max_ratio decimal default 1000.0) RETURNS random_campaign AS $$ - SELECT * FROM ( - SELECT DISTINCT ON (CASE WHEN parent_id IS NOT NULL THEN parent_id ELSE entity_id END) - banner_id, url, entity_id, affiliate_id, parent_id, id - FROM ( - SELECT - campaigns.*, entities.parent_id as parent_id - FROM campaigns - LEFT JOIN entities ON entities.id = campaigns.entity_id - LEFT JOIN banners ON banners.id = campaigns.banner_id - WHERE banner_id IS NOT NULL - AND ratio >= min_ratio - AND ratio <= max_ratio - ORDER BY RANDOM() - ) random_campaigns - ) random_banners - ORDER BY RANDOM() - LIMIT 1; - $$ LANGUAGE SQL STABLE; - `); -}; - -exports.down = async (knex) => { - await knex.schema.alterTable('random_campaign', (table) => { - table.dropColumn('campaign_id'); - }); - - await knex.raw(` - CREATE OR REPLACE FUNCTION get_random_campaign(min_ratio decimal default 0, max_ratio decimal default 1000.0) RETURNS random_campaign AS $$ - SELECT * FROM ( - SELECT DISTINCT ON (CASE WHEN parent_id IS NOT NULL THEN parent_id ELSE entity_id END) - banner_id, url, entity_id, affiliate_id, parent_id - FROM ( - SELECT - campaigns.*, entities.parent_id as parent_id - FROM campaigns - LEFT JOIN entities ON entities.id = campaigns.entity_id - LEFT JOIN banners ON banners.id = campaigns.banner_id - WHERE banner_id IS NOT NULL - AND ratio >= min_ratio - AND ratio <= max_ratio - ORDER BY RANDOM() - ) random_campaigns - ) random_banners - ORDER BY RANDOM() - LIMIT 1; - $$ LANGUAGE SQL STABLE; - `); -}; diff --git a/migrations/20230709231430_release_summaries.js b/migrations/20230709231430_release_summaries.js deleted file mode 100644 index 7ba20812..00000000 --- a/migrations/20230709231430_release_summaries.js +++ /dev/null @@ -1,47 +0,0 @@ -const config = require('config'); - -exports.up = async (knex) => { - await knex.raw(` - CREATE MATERIALIZED VIEW releases_summaries AS ( - SELECT - releases.id as release_id, - channels.slug as channel_slug, - channels.type as channel_type, - networks.slug as network_slug, - networks.type as network_type, - parent_networks.slug as parent_network_slug, - parent_networks.type as parent_network_type, - studios.showcased IS NOT false - AND (channels.showcased IS NOT false OR COALESCE(studios.showcased, false) = true) - AND (networks.showcased IS NOT false OR COALESCE(channels.showcased, false) = true OR COALESCE(studios.showcased, false) = true) - AS showcased, - batches.showcased AS batch_showcased, - releases.effective_date, - releases.created_at, - array_agg(tags.slug ORDER BY tags.priority DESC) FILTER (WHERE tags.slug IS NOT NULL) AS tags - FROM releases - LEFT JOIN releases_tags ON releases_tags.release_id = releases.id - LEFT JOIN tags ON tags.id = releases_tags.tag_id - LEFT JOIN entities AS channels ON channels.id = releases.entity_id - LEFT JOIN entities AS studios ON studios.id = releases.studio_id - LEFT JOIN entities AS networks ON networks.id = channels.parent_id - LEFT JOIN entities AS parent_networks ON parent_networks.id = networks.parent_id - LEFT JOIN batches ON batches.id = releases.updated_batch_id - GROUP BY releases.id, studios.showcased, batches.showcased, - channels.showcased, channels.slug, channels.type, - networks.showcased, networks.slug, networks.type, - parent_networks.slug, parent_networks.type - ); - - COMMENT ON MATERIALIZED VIEW releases_summaries IS E'@foreignKey (release_id) references releases (id)'; - GRANT ALL ON releases_summaries TO :visitor; - `, { - visitor: knex.raw(config.database.query.user), - }); -}; - -exports.down = async (knex) => { - await knex.raw(` - DROP MATERIALIZED VIEW IF EXISTS releases_summaries; - `); -}; diff --git a/migrations/20230723001912_actors_social_entity.js b/migrations/20230723001912_actors_social_entity.js deleted file mode 100644 index 29acdc1a..00000000 --- a/migrations/20230723001912_actors_social_entity.js +++ /dev/null @@ -1,27 +0,0 @@ -exports.up = async (knex) => { - await knex.schema.alterTable('actors_social', (table) => { - table.integer('profile_id') - .references('id') - .inTable('actors_profiles'); - - table.dropUnique(['url', 'actor_id']); - table.unique(['url', 'actor_id', 'profile_id']); - }); - - await knex.raw(` - CREATE UNIQUE INDEX actors_social_url_actor_id_null_unique ON actors_social (url, actor_id) WHERE profile_id IS NULL; - `); -}; - -exports.down = async (knex) => { - await knex.raw(` - DROP INDEX actors_social_url_actor_id_null_unique; - `); - - await knex.schema.alterTable('actors_social', (table) => { - table.dropUnique(['url', 'actor_id', 'profile_id']); - table.unique(['url', 'actor_id']); - - table.dropColumn('profile_id'); - }); -}; diff --git a/migrations/20230725001453_caps.js b/migrations/20230725001453_caps.js deleted file mode 100644 index 95a85550..00000000 --- a/migrations/20230725001453_caps.js +++ /dev/null @@ -1,24 +0,0 @@ -const config = require('config'); - -exports.up = async (knex) => { - await knex.schema.createTable('releases_caps', (table) => { - table.integer('release_id') - .notNullable() - .references('id') - .inTable('releases') - .onDelete('cascade'); - - table.text('media_id') - .notNullable() - .references('id') - .inTable('media'); - }); - - await knex.raw('GRANT ALL ON releases_caps TO :visitor;', { - visitor: knex.raw(config.database.query.user), - }); -}; - -exports.down = async (knex) => { - await knex.schema.dropTable('releases_caps'); -}; diff --git a/migrations/20230725020639_blood_type.js b/migrations/20230725020639_blood_type.js deleted file mode 100644 index af1a36d9..00000000 --- a/migrations/20230725020639_blood_type.js +++ /dev/null @@ -1,27 +0,0 @@ -exports.up = async (knex) => { - await knex.schema.alterTable('actors_profiles', (table) => { - table.string('hair_type'); - table.decimal('shoe_size'); - table.string('blood_type'); - }); - - await knex.schema.alterTable('actors', (table) => { - table.string('hair_type'); - table.decimal('shoe_size'); - table.string('blood_type'); - }); -}; - -exports.down = async (knex) => { - await knex.schema.alterTable('actors_profiles', (table) => { - table.dropColumn('hair_type'); - table.dropColumn('shoe_size'); - table.dropColumn('blood_type'); - }); - - await knex.schema.alterTable('actors', (table) => { - table.dropColumn('hair_type'); - table.dropColumn('shoe_size'); - table.dropColumn('blood_type'); - }); -}; diff --git a/migrations/20230725031547_stealth_batch.js b/migrations/20230725031547_stealth_batch.js deleted file mode 100644 index 3aba478d..00000000 --- a/migrations/20230725031547_stealth_batch.js +++ /dev/null @@ -1,13 +0,0 @@ -exports.up = async (knex) => { - await knex.schema.alterTable('batches', (table) => { - table.boolean('showcased') - .notNullable() - .defaultTo(true); - }); -}; - -exports.down = async (knex) => { - await knex.schema.alterTable('batches', (table) => { - table.dropColumn('showcased'); - }); -}; diff --git a/migrations/20230801225038_scene_titles.js b/migrations/20230801225038_scene_titles.js deleted file mode 100644 index f69154ad..00000000 --- a/migrations/20230801225038_scene_titles.js +++ /dev/null @@ -1,27 +0,0 @@ -exports.up = async (knex) => { - await knex.schema.alterTable('releases', (table) => { - table.specificType('alt_titles', 'text ARRAY'); - }); - - await knex.schema.alterTable('movies', (table) => { - table.specificType('alt_titles', 'text ARRAY'); - }); - - await knex.schema.alterTable('series', (table) => { - table.specificType('alt_titles', 'text ARRAY'); - }); -}; - -exports.down = async (knex) => { - await knex.schema.alterTable('releases', (table) => { - table.dropColumn('alt_titles'); - }); - - await knex.schema.alterTable('movies', (table) => { - table.dropColumn('alt_titles'); - }); - - await knex.schema.alterTable('series', (table) => { - table.dropColumn('alt_titles'); - }); -}; diff --git a/migrations/20230811015424_teasers.js b/migrations/20230811015424_teasers.js deleted file mode 100644 index 4082a071..00000000 --- a/migrations/20230811015424_teasers.js +++ /dev/null @@ -1,36 +0,0 @@ -exports.up = async (knex) => { - await knex.schema.createTable('movies_teasers', (table) => { - table.integer('movie_id', 16) - .notNullable() - .references('id') - .inTable('movies') - .onDelete('cascade'); - - table.text('media_id', 21) - .notNullable() - .references('id') - .inTable('media'); - - table.unique('movie_id'); - }); - - await knex.schema.createTable('series_teasers', (table) => { - table.integer('serie_id', 16) - .notNullable() - .references('id') - .inTable('series') - .onDelete('cascade'); - - table.text('media_id', 21) - .notNullable() - .references('id') - .inTable('media'); - - table.unique('serie_id'); - }); -}; - -exports.down = async (knex) => { - await knex.schema.dropTable('movies_teasers'); - await knex.schema.dropTable('series_teasers'); -}; diff --git a/migrations/20231107044032_photocount.js b/migrations/20231107044032_photocount.js deleted file mode 100644 index cb065c94..00000000 --- a/migrations/20231107044032_photocount.js +++ /dev/null @@ -1,19 +0,0 @@ -exports.up = async (knex) => { - await knex.schema.alterTable('releases', (table) => { - table.integer('photo_count'); - }); - - await knex.schema.alterTable('movies', (table) => { - table.integer('photo_count'); - }); -}; - -exports.down = async (knex) => { - await knex.schema.alterTable('releases', (table) => { - table.dropColumn('photo_count'); - }); - - await knex.schema.alterTable('movies', (table) => { - table.dropColumn('photo_count'); - }); -}; diff --git a/migrations/20231109011513_alert_andor_pattern.js b/migrations/20231109011513_alert_andor_pattern.js deleted file mode 100644 index 7de1f1e6..00000000 --- a/migrations/20231109011513_alert_andor_pattern.js +++ /dev/null @@ -1,42 +0,0 @@ -const config = require('config'); - -exports.up = async (knex) => { - await knex.schema.alterTable('alerts', (table) => { - table.boolean('all') - .defaultTo(true); - }); - - await knex.schema.alterTable('alerts_entities', (table) => { - table.dropUnique('alert_id'); - }); - - await knex.schema.createTable('alerts_matches', (table) => { - table.increments('id'); - - table.integer('alert_id') - .references('id') - .inTable('alerts') - .onDelete('cascade'); - - table.string('property'); - table.string('expression'); - }); - - await knex.raw(` - GRANT SELECT ON alerts_matches TO :visitor; - `, { - visitor: knex.raw(config.database.query.user), - }); -}; - -exports.down = async (knex) => { - await knex.schema.alterTable('alerts', (table) => { - table.dropColumn('all'); - }); - - await knex.schema.alterTable('alerts_entities', (table) => { - table.unique('alert_id'); - }); - - await knex.schema.dropTable('alerts_matches'); -}; diff --git a/migrations/20231201040310_tagsplit.js b/migrations/20231201040310_tagsplit.js deleted file mode 100644 index 4ffedc33..00000000 --- a/migrations/20231201040310_tagsplit.js +++ /dev/null @@ -1,21 +0,0 @@ -exports.up = async (knex) => { - await knex.schema.alterTable('tags', (table) => { - table.specificType('implied_tag_ids', 'integer[]'); - }); - - await knex.schema.alterTable('releases_tags', (table) => { - table.enum('source', ['scraper', 'editor', 'implied']) - .notNullable() - .defaultTo('scraper'); - }); -}; - -exports.down = async (knex) => { - await knex.schema.alterTable('tags', (table) => { - table.dropColumn('implied_tag_ids'); - }); - - await knex.schema.alterTable('releases_tags', (table) => { - table.dropColumn('source'); - }); -}; diff --git a/migrations/20231216015820_movies_series_photo_count.js b/migrations/20231216015820_movies_series_photo_count.js deleted file mode 100644 index 994ae39a..00000000 --- a/migrations/20231216015820_movies_series_photo_count.js +++ /dev/null @@ -1,19 +0,0 @@ -exports.up = async (knex) => { - await knex.schema.alterTable('movies', (table) => { - table.integer('photo_count'); - }); - - await knex.schema.alterTable('series', (table) => { - table.integer('photo_count'); - }); -}; - -exports.down = async (knex) => { - await knex.schema.alterTable('movies', (table) => { - table.dropColumn('photo_count'); - }); - - await knex.schema.alterTable('series', (table) => { - table.dropColumn('photo_count'); - }); -}; diff --git a/migrations/20240104234936_actors_metadata.js b/migrations/20240104234936_actors_metadata.js deleted file mode 100644 index 87e6a422..00000000 --- a/migrations/20240104234936_actors_metadata.js +++ /dev/null @@ -1,62 +0,0 @@ -const config = require('config'); - -exports.up = async function up(knex) { - await knex.raw(` - CREATE MATERIALIZED VIEW actors_meta AS ( - SELECT - actors.id as actor_id, - COUNT(DISTINCT stashes_actors)::integer as stashed, - COUNT(DISTINCT releases_actors)::integer as scenes, - row_to_json(avatars) as avatar - FROM actors - LEFT JOIN stashes_actors ON stashes_actors.actor_id = actors.id - LEFT JOIN releases_actors ON releases_actors.actor_id = actors.id - LEFT JOIN media AS avatars ON avatars.id = actors.avatar_media_id - GROUP BY - actors.id, - avatars.id - ); - - CREATE MATERIALIZED VIEW scenes_meta AS ( - SELECT - releases.id as scene_id, - COUNT(DISTINCT stashes_scenes)::integer as stashed - FROM releases - LEFT JOIN stashes_scenes ON stashes_scenes.scene_id = releases.id - GROUP BY releases.id - ); - - CREATE MATERIALIZED VIEW movies_meta AS ( - SELECT - movie_id, - stashed, - stashed_scenes, - stashed + stashed_scenes as stashed_total - FROM ( - SELECT - movies.id as movie_id, - COUNT(DISTINCT stashes_movies)::integer as stashed, - COUNT(DISTINCT stashes_scenes)::integer as stashed_scenes - FROM movies - LEFT JOIN stashes_movies ON stashes_movies.movie_id = movies.id - LEFT JOIN movies_scenes ON movies_scenes.movie_id = movies.id - LEFT JOIN stashes_scenes ON stashes_scenes.scene_id = movies_scenes.scene_id - GROUP BY movies.id - ) AS meta - ); - - GRANT ALL ON actors_meta TO :visitor; - GRANT ALL ON scenes_meta TO :visitor; - GRANT ALL ON movies_meta TO :visitor; - `, { - visitor: knex.raw(config.database.query.user), - }); -}; - -exports.down = async function down(knex) { - await knex.raw(` - DROP MATERIALIZED VIEW IF EXISTS actors_meta; - DROP MATERIALIZED VIEW IF EXISTS scenes_meta; - DROP MATERIALIZED VIEW IF EXISTS movies_meta; - `); -}; diff --git a/migrations/20240229054247_users_case.js b/migrations/20240229054247_users_case.js deleted file mode 100644 index 2ca2ab3f..00000000 --- a/migrations/20240229054247_users_case.js +++ /dev/null @@ -1,21 +0,0 @@ -exports.up = async (knex) => { - await knex.schema.alterTable('users', (table) => { - table.dropUnique('username'); - }); - - await knex.raw(` - CREATE UNIQUE INDEX username_unique_index ON users (LOWER(username)); - CREATE UNIQUE INDEX email_unique_index ON users (LOWER(email)); - `); -}; - -exports.down = async (knex) => { - await knex.raw(` - DROP INDEX IF EXISTS username_unique_index; - DROP INDEX IF EXISTS email_unique_index; - `); - - await knex.schema.alterTable('users', (table) => { - table.unique('username'); - }); -}; diff --git a/migrations/20240303020146_stashes_meta.js b/migrations/20240303020146_stashes_meta.js deleted file mode 100644 index bff5e55f..00000000 --- a/migrations/20240303020146_stashes_meta.js +++ /dev/null @@ -1,22 +0,0 @@ -exports.up = async function up(knex) { - await knex.raw(` - CREATE MATERIALIZED VIEW stashes_meta AS ( - SELECT - stashes.id as stash_id, - COUNT(DISTINCT stashes_scenes)::integer as stashed_scenes, - COUNT(DISTINCT stashes_movies)::integer as stashed_movies, - COUNT(DISTINCT stashes_actors)::integer as stashed_actors - FROM stashes - LEFT JOIN stashes_scenes ON stashes_scenes.stash_id = stashes.id - LEFT JOIN stashes_movies ON stashes_movies.stash_id = stashes.id - LEFT JOIN stashes_actors ON stashes_actors.stash_id = stashes.id - GROUP BY stashes.id - ); - `); -}; - -exports.down = async function down(knex) { - await knex.raw(` - DROP MATERIALIZED VIEW IF EXISTS stashes_meta; - `); -}; diff --git a/migrations/20240414003818_alerts_any.js b/migrations/20240414003818_alerts_any.js deleted file mode 100644 index a3632257..00000000 --- a/migrations/20240414003818_alerts_any.js +++ /dev/null @@ -1,38 +0,0 @@ -exports.up = async (knex) => { - await knex.schema.alterTable('alerts', (table) => { - table.boolean('all_actors') - .notNullable() - .defaultTo(true); - - table.boolean('all_entities') - .notNullable() - .defaultTo(true); - - table.boolean('all_tags') - .notNullable() - .defaultTo(true); - - table.boolean('all_matches') - .notNullable() - .defaultTo(true); - }); - - await knex.raw(` - UPDATE alerts - SET - all_actors = false, - all_entities = false, - all_tags = false, - all_matches= false - WHERE alerts.all = false; - `); -}; - -exports.down = async (knex) => { - await knex.schema.alterTable('alerts', (table) => { - table.dropColumn('all_actors'); - table.dropColumn('all_entities'); - table.dropColumn('all_tags'); - table.dropColumn('all_matches'); - }); -}; diff --git a/migrations/20240815013526_entities_options.js b/migrations/20240815013526_entities_options.js deleted file mode 100644 index f4eb899a..00000000 --- a/migrations/20240815013526_entities_options.js +++ /dev/null @@ -1,65 +0,0 @@ -const config = require('config'); - -exports.up = async (knex) => { - await knex.schema.alterTable('entities', (table) => { - // internal options, as opposed to parameters for scraper options - table.json('options'); - }); - - await knex.schema.alterTable('releases', (table) => { - table.dropForeign('entity_id'); - - table.foreign('entity_id') - .references('id') - .inTable('entities') - .onDelete('cascade'); - }); - - await knex.schema.alterTable('releases_caps', (table) => { - table.unique(['release_id', 'media_id']); - }); - - await knex.schema.createTable('movies_tags', (table) => { - table.integer('tag_id') - .references('id') - .inTable('tags'); - - table.integer('movie_id') - .notNullable() - .references('id') - .inTable('movies') - .onDelete('cascade'); - - table.text('original_tag'); - - table.text('source') - .defaultTo('scraper'); - - table.unique(['tag_id', 'movie_id']); - }); - - await knex.raw('GRANT ALL ON ALL TABLES IN SCHEMA public TO :visitor;', { - visitor: knex.raw(config.database.query.user), - }); -}; - -exports.down = async (knex) => { - await knex.schema.alterTable('entities', (table) => { - table.dropColumn('options'); - }); - - await knex.schema.alterTable('releases', (table) => { - table.dropForeign('entity_id'); - - table.foreign('entity_id') - .references('id') - .inTable('entities') - .onDelete('no action'); - }); - - await knex.schema.alterTable('releases_caps', (table) => { - table.dropUnique(['release_id', 'media_id']); - }); - - await knex.schema.dropTable('movies_tags'); -}; diff --git a/migrations/20240822005656_entity_tags_cascade.js b/migrations/20240822005656_entity_tags_cascade.js deleted file mode 100644 index 071c1563..00000000 --- a/migrations/20240822005656_entity_tags_cascade.js +++ /dev/null @@ -1,31 +0,0 @@ -exports.up = async (knex) => { - await knex.schema.alterTable('entities_tags', (table) => { - table.dropForeign('tag_id'); - table.dropForeign('entity_id'); - - table.foreign('tag_id') - .references('id') - .inTable('tags') - .onDelete('cascade'); - - table.foreign('entity_id') - .references('id') - .inTable('entities') - .onDelete('cascade'); - }); -}; - -exports.down = async (knex) => { - await knex.schema.alterTable('entities_tags', (table) => { - table.dropForeign('tag_id'); - table.dropForeign('entity_id'); - - table.foreign('tag_id') - .references('id') - .inTable('tags'); - - table.foreign('entity_id') - .references('id') - .inTable('entities'); - }); -}; diff --git a/migrations/20240825232707_summaries.js b/migrations/20240825232707_summaries.js deleted file mode 100644 index d26c01d3..00000000 --- a/migrations/20240825232707_summaries.js +++ /dev/null @@ -1,25 +0,0 @@ -exports.up = async (knex) => { - await knex.schema.createTable('users_templates', (table) => { - table.increments('id'); - - table.integer('user_id') - .notNullable() - .references('id') - .inTable('users'); - - table.string('name') - .notNullable(); - - table.text('template') - .notNullable(); - - table.unique(['user_id', 'name']); - - table.datetime('created_at') - .defaultTo(knex.fn.now()); - }); -}; - -exports.down = async (knex) => { - await knex.schema.dropTable('users_templates'); -}; diff --git a/migrations/20240830234355_api_keys.js b/migrations/20240830234355_api_keys.js deleted file mode 100644 index b649930f..00000000 --- a/migrations/20240830234355_api_keys.js +++ /dev/null @@ -1,28 +0,0 @@ -exports.up = async (knex) => { - await knex.schema.createTable('users_keys', (table) => { - table.increments('id'); - - table.integer('user_id') - .notNullable() - .references('id') - .inTable('users'); - - table.text('key') - .notNullable(); - - table.string('identifier'); - - table.unique(['user_id', 'identifier']); - - table.datetime('last_used_at'); - table.specificType('last_used_ip', 'inet'); - - table.datetime('created_at') - .notNullable() - .defaultTo(knex.fn.now()); - }); -}; - -exports.down = async (knex) => { - await knex.schema.dropTable('users_keys'); -}; diff --git a/migrations/20240904234305_content_versions.js b/migrations/20240904234305_content_versions.js deleted file mode 100644 index b79fa283..00000000 --- a/migrations/20240904234305_content_versions.js +++ /dev/null @@ -1,87 +0,0 @@ -exports.up = async (knex) => { - await knex.schema.createTable('scenes_revisions', (table) => { - table.increments('id'); - - table.integer('scene_id') - .notNullable() - .references('id') - .inTable('releases') - .onDelete('set null'); - - table.integer('user_id') - .references('id') - .inTable('users') - .onDelete('set null'); - - table.json('base') - .notNullable(); - - table.json('deltas') - .notNullable(); - - table.text('hash') - .notNullable(); - - table.text('comment'); - - table.boolean('approved'); - - table.integer('reviewed_by') - .references('id') - .inTable('users') - .onDelete('set null'); - - table.datetime('reviewed_at'); - table.text('feedback'); - - table.datetime('applied_at'); - - table.datetime('created_at') - .notNullable() - .defaultTo(knex.fn.now()); - }); - - await knex.schema.createTable('bans', (table) => { - table.increments('id'); - - table.integer('user_id') - .references('id') - .inTable('users') - .onDelete('set null'); - - table.string('username'); - table.specificType('ip', 'cidr'); - - table.boolean('match_all') - .notNullable() - .defaultTo(false); - - table.string('scope'); - table.boolean('shadow'); - - table.integer('banned_by') - .references('id') - .inTable('users') - .onDelete('set null'); - - table.datetime('expires_at') - .notNullable(); - - table.datetime('created_at') - .notNullable() - .defaultTo(knex.fn.now()); - }); - - await knex.schema.alterTable('users', (table) => { - table.specificType('last_ip', 'cidr'); - }); -}; - -exports.down = async (knex) => { - await knex.schema.dropTable('scenes_revisions'); - await knex.schema.dropTable('bans'); - - await knex.schema.alterTable('users', (table) => { - table.dropColumn('last_ip'); - }); -}; diff --git a/migrations/20241016020256_profile_details.js b/migrations/20241016020256_profile_details.js deleted file mode 100644 index 76e7fa6e..00000000 --- a/migrations/20241016020256_profile_details.js +++ /dev/null @@ -1,35 +0,0 @@ -exports.up = async (knex) => { - await knex.schema.alterTable('actors', (table) => { - table.integer('leg'); - table.integer('foot'); - table.integer('thigh'); - }); - - await knex.schema.alterTable('actors_profiles', (table) => { - table.integer('leg'); - table.integer('foot'); - table.integer('thigh'); - }); - - await knex.schema.alterTable('releases', (table) => { - table.integer('video_count'); - }); -}; - -exports.down = async (knex) => { - await knex.schema.alterTable('actors', (table) => { - table.dropColumn('leg'); - table.dropColumn('foot'); - table.dropColumn('thigh'); - }); - - await knex.schema.alterTable('actors_profiles', (table) => { - table.dropColumn('leg'); - table.dropColumn('foot'); - table.dropColumn('thigh'); - }); - - await knex.schema.alterTable('releases', (table) => { - table.dropColumn('video_count'); - }); -}; diff --git a/migrations/20241016171648_revision_nullable.js b/migrations/20241016171648_revision_nullable.js deleted file mode 100644 index 422fe559..00000000 --- a/migrations/20241016171648_revision_nullable.js +++ /dev/null @@ -1,15 +0,0 @@ -exports.up = async (knex) => { - await knex.schema.alterTable('scenes_revisions', (table) => { - table.integer('scene_id') - .nullable() - .alter(); - }); -}; - -exports.down = async (knex) => { - await knex.schema.alterTable('scenes_revisions', (table) => { - table.integer('scene_id') - .notNullable() - .alter(); - }); -}; diff --git a/migrations/20241019013040_actor_revisions.js b/migrations/20241019013040_actor_revisions.js deleted file mode 100644 index 6b969262..00000000 --- a/migrations/20241019013040_actor_revisions.js +++ /dev/null @@ -1,119 +0,0 @@ -exports.up = async (knex) => { - await knex.raw('CREATE UNIQUE INDEX unique_main_profiles ON actors_profiles (actor_id) WHERE (entity_id IS NULL);'); - - await knex.schema.createTable('actors_revisions', (table) => { - table.increments('id'); - - table.integer('actor_id') - .references('id') - .inTable('actors') - .onDelete('set null'); - - table.integer('profile_id') - .references('id') - .inTable('actors_profiles') - .onDelete('set null'); - - table.integer('user_id') - .references('id') - .inTable('users') - .onDelete('set null'); - - table.json('base') - .notNullable(); - - table.json('deltas') - .notNullable(); - - table.text('hash') - .notNullable(); - - table.text('comment'); - - table.boolean('approved'); - - table.integer('reviewed_by') - .references('id') - .inTable('users') - .onDelete('set null'); - - table.datetime('reviewed_at'); - table.text('feedback'); - - table.datetime('applied_at'); - - table.datetime('created_at') - .notNullable() - .defaultTo(knex.fn.now()); - }); - - await knex.schema.alterTable('actors', (table) => { - table.integer('boobs_volume'); - table.enum('boobs_implant', ['saline', 'silicone', 'gummy', 'fat']); - table.enum('boobs_placement', ['over', 'under']); - table.string('boobs_surgeon'); - - table.boolean('natural_butt'); - table.integer('butt_volume'); - table.enum('butt_implant', ['bbl', 'lift', 'silicone', 'lipo', 'filler', 'mms']); - - table.boolean('natural_lips'); - table.integer('lips_volume'); - - table.string('agency'); - }); - - await knex.schema.alterTable('actors_profiles', (table) => { - table.integer('boobs_volume'); - table.enum('boobs_implant', ['saline', 'silicone', 'gummy', 'fat']); - table.enum('boobs_placement', ['over', 'under']); - table.string('boobs_surgeon'); - - table.boolean('natural_butt'); - table.integer('butt_volume'); - table.enum('butt_implant', ['bbl', 'lift', 'silicone', 'lipo', 'filler', 'mms']); - - table.boolean('natural_lips'); - table.integer('lips_volume'); - - table.string('agency'); - }); -}; - -exports.down = async (knex) => { - await knex.raw('DROP INDEX unique_main_profiles;'); - - await knex.schema.dropTable('actors_revisions'); - - await knex.schema.alterTable('actors', (table) => { - table.dropColumn('boobs_volume'); - table.dropColumn('boobs_implant'); - table.dropColumn('boobs_placement'); - table.dropColumn('boobs_surgeon'); - - table.dropColumn('natural_butt'); - table.dropColumn('butt_volume'); - table.dropColumn('butt_implant'); - - table.dropColumn('natural_lips'); - table.dropColumn('lips_volume'); - - table.dropColumn('agency'); - }); - - await knex.schema.alterTable('actors_profiles', (table) => { - table.dropColumn('boobs_volume'); - table.dropColumn('boobs_implant'); - table.dropColumn('boobs_placement'); - table.dropColumn('boobs_surgeon'); - - table.dropColumn('natural_butt'); - table.dropColumn('butt_volume'); - table.dropColumn('butt_implant'); - - table.dropColumn('natural_lips'); - table.dropColumn('lips_volume'); - - table.dropColumn('agency'); - }); -}; diff --git a/migrations/20241024182155_actor_fields.js b/migrations/20241024182155_actor_fields.js deleted file mode 100644 index e50018c0..00000000 --- a/migrations/20241024182155_actor_fields.js +++ /dev/null @@ -1,19 +0,0 @@ -function createColumns(table) { - table.enum('boobs_incision', ['mammary', 'areolar', 'crescent', 'lollipop', 'anchor', 'axillary', 'umbilical']); - table.boolean('natural_labia'); -} - -exports.up = async (knex) => { - await knex.schema.alterTable('actors', createColumns); - await knex.schema.alterTable('actors_profiles', createColumns); -}; - -function dropColumns(table) { - table.dropColumn('boobs_incision'); - table.dropColumn('natural_labia'); -} - -exports.down = async (knex) => { - await knex.schema.alterTable('actors', dropColumns); - await knex.schema.alterTable('actors_profiles', dropColumns); -}; diff --git a/migrations/20241025174046_actors_avatars.js b/migrations/20241025174046_actors_avatars.js deleted file mode 100644 index 32132e81..00000000 --- a/migrations/20241025174046_actors_avatars.js +++ /dev/null @@ -1,75 +0,0 @@ -exports.up = async function(knex) { - // restore avatars in table in case of rollback and rerun - const avatars = await knex('actors_avatars') - .select('actors_avatars.*', 'actors_profiles.actor_id') - .leftJoin('actors_profiles', 'actors_profiles.id', 'actors_avatars.profile_id'); - - await knex('actors_avatars').delete(); - - await knex.schema.alterTable('actors_avatars', (table) => { - table.integer('profile_id') - .nullable() - .alter(); - - table.integer('actor_id') - .notNullable() - .references('id') - .inTable('actors'); - - table.datetime('created_at') - .notNullable() - .defaultTo(knex.fn.now()); - - table.dropUnique('profile_id'); - table.unique(['profile_id', 'media_id']); - }); - - await knex.schema.alterTable('media', (table) => { - // actor avatars often retain the same URL when updated, handle URL-deduping in app code - table.dropUnique('source'); - table.string('source_version'); // usually etag - }); - - await knex.raw('CREATE UNIQUE INDEX unique_main_avatars ON actors_avatars (actor_id) WHERE (profile_id IS NULL);'); - - if (avatars.length > 0) { - await knex('actors_avatars').insert(avatars); - } - - const profiles = await knex('actors_profiles') - .select('id', 'actor_id', 'avatar_media_id') - .whereNotNull('avatar_media_id'); - - await knex('actors_avatars') - .insert(profiles.map((profile) => ({ - actor_id: profile.actor_id, - profile_id: profile.id, - media_id: profile.avatar_media_id, - }))) - .onConflict() - .ignore(); -}; - -exports.down = async function(knex) { - // no need to delete all entries, only the ones incompatible with the old scheme - await knex('actors_avatars') - .whereNull('profile_id') - .delete(); - - await knex.schema.alterTable('actors_avatars', (table) => { - table.integer('profile_id') - .notNullable() - .alter(); - - table.dropColumn('actor_id'); - table.dropColumn('created_at'); - - table.unique('profile_id'); - table.dropUnique(['profile_id', 'media_id']); - }); - - await knex.schema.alterTable('media', (table) => { - table.dropColumn('source_version'); - table.unique('source'); - }); -}; diff --git a/migrations/20241103060644_socials.js b/migrations/20241103060644_socials.js deleted file mode 100644 index 11eeafb2..00000000 --- a/migrations/20241103060644_socials.js +++ /dev/null @@ -1,46 +0,0 @@ -exports.up = async (knex) => { - await knex.schema.alterTable('actors_social', (table) => { - table.dropUnique(['url', 'actor_id', 'profile_id']); - table.dropColumn('profile_id'); - - table.string('handle'); - - table.boolean('is_broken') - .notNullable() - .defaultTo(false); - - table.datetime('pinged_at'); - table.datetime('verified_at'); - - table.unique(['actor_id', 'platform', 'handle']); - table.unique(['actor_id', 'url']); - }); - - await knex.raw('ALTER TABLE actors_social ADD CONSTRAINT socials_url_or_handle CHECK (num_nulls(handle, url) = 1);'); - await knex.raw('ALTER TABLE actors_social ADD CONSTRAINT socials_handle_and_platform CHECK (num_nulls(platform, handle) = 2 or num_nulls(platform, handle) = 0);'); - - await knex.schema.renameTable('actors_social', 'actors_socials'); -}; - -exports.down = async (knex) => { - await knex.raw('ALTER TABLE actors_socials DROP CONSTRAINT socials_url_or_handle;'); - await knex.raw('ALTER TABLE actors_socials DROP CONSTRAINT socials_handle_and_platform;'); - - await knex.schema.renameTable('actors_socials', 'actors_social'); - - await knex.schema.alterTable('actors_social', (table) => { - table.dropUnique(['actor_id', 'platform', 'handle']); - table.dropUnique(['actor_id', 'url']); - - table.integer('profile_id') - .references('id') - .inTable('actors_profiles'); - - table.dropColumn('handle'); - table.dropColumn('verified_at'); - table.dropColumn('pinged_at'); - table.dropColumn('is_broken'); - - table.unique(['url', 'actor_id', 'profile_id']); - }); -}; diff --git a/migrations/20250304025013_media_indexes.js b/migrations/20250304025013_media_indexes.js deleted file mode 100644 index a03a9cba..00000000 --- a/migrations/20250304025013_media_indexes.js +++ /dev/null @@ -1,53 +0,0 @@ -exports.up = async (knex) => { - await knex.schema.alterTable('media', (table) => table.index('sfw_media_id')); - await knex.schema.alterTable('actors_profiles', (table) => table.index('avatar_media_id')); - await knex.schema.alterTable('actors_avatars', (table) => table.index('media_id')); - await knex.schema.alterTable('actors_photos', (table) => table.index('media_id')); - await knex.schema.alterTable('chapters_photos', (table) => table.index('media_id')); - await knex.schema.alterTable('chapters_posters', (table) => table.index('media_id')); - await knex.schema.alterTable('movies_covers', (table) => table.index('media_id')); - await knex.schema.alterTable('movies_photos', (table) => table.index('media_id')); - await knex.schema.alterTable('movies_posters', (table) => table.index('media_id')); - await knex.schema.alterTable('movies_teasers', (table) => table.index('media_id')); - await knex.schema.alterTable('movies_trailers', (table) => table.index('media_id')); - await knex.schema.alterTable('releases_caps', (table) => table.index('media_id')); - await knex.schema.alterTable('releases_covers', (table) => table.index('media_id')); - await knex.schema.alterTable('releases_posters', (table) => table.index('media_id')); - await knex.schema.alterTable('releases_photos', (table) => table.index('media_id')); - await knex.schema.alterTable('releases_teasers', (table) => table.index('media_id')); - await knex.schema.alterTable('releases_trailers', (table) => table.index('media_id')); - await knex.schema.alterTable('series_covers', (table) => table.index('media_id')); - await knex.schema.alterTable('series_photos', (table) => table.index('media_id')); - await knex.schema.alterTable('series_posters', (table) => table.index('media_id')); - await knex.schema.alterTable('series_teasers', (table) => table.index('media_id')); - await knex.schema.alterTable('series_trailers', (table) => table.index('media_id')); - await knex.schema.alterTable('tags_photos', (table) => table.index('media_id')); - await knex.schema.alterTable('tags_posters', (table) => table.index('media_id')); -}; - -exports.down = async (knex) => { - await knex.schema.alterTable('media', (table) => table.dropIndex('sfw_media_id')); - await knex.schema.alterTable('actors_profiles', (table) => table.dropIndex('avatar_media_id')); - await knex.schema.alterTable('actors_avatars', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('actors_photos', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('chapters_photos', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('chapters_posters', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('movies_covers', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('movies_photos', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('movies_posters', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('movies_teasers', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('movies_trailers', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('releases_caps', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('releases_covers', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('releases_posters', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('releases_photos', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('releases_teasers', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('releases_trailers', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('series_covers', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('series_photos', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('series_posters', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('series_teasers', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('series_trailers', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('tags_photos', (table) => table.dropIndex('media_id')); - await knex.schema.alterTable('tags_posters', (table) => table.dropIndex('media_id')); -}; diff --git a/migrations/20250307030844_quick_alerts.js b/migrations/20250307030844_quick_alerts.js deleted file mode 100644 index 5c9978d6..00000000 --- a/migrations/20250307030844_quick_alerts.js +++ /dev/null @@ -1,102 +0,0 @@ -const config = require('config'); - -exports.up = async (knex) => { - await knex.schema.alterTable('alerts', (table) => { - table.boolean('from_preset') - .notNullable() - .defaultTo(false); - - table.text('comment'); - }); - - await knex.schema.createMaterializedView('alerts_users_actors', (view) => { - view.columns('user_id', 'actor_id', 'alert_ids'); - - view.as( - knex('alerts_actors') - .select( - 'alerts.user_id', - 'alerts_actors.actor_id', - knex.raw('array_agg(distinct alerts.id) as alert_ids'), - knex.raw('(alerts_tags.id is null and alerts_entities.id is null and alerts_matches.id is null and related_actors.id is null) as is_only'), - ) - .leftJoin('alerts', 'alerts.id', 'alerts_actors.alert_id') - .leftJoin('alerts_entities', 'alerts_entities.alert_id', 'alerts_actors.alert_id') - .leftJoin('alerts_tags', 'alerts_tags.alert_id', 'alerts_actors.alert_id') - .leftJoin('alerts_matches', 'alerts_matches.alert_id', 'alerts_actors.alert_id') - .leftJoin('alerts_actors as related_actors', (joinBuilder) => { - joinBuilder - .on('related_actors.alert_id', 'alerts_actors.alert_id') - .on('related_actors.actor_id', '!=', 'alerts_actors.actor_id'); - }) - .groupBy(['user_id', 'alerts_actors.actor_id', 'is_only']), - ); - }); - - await knex.schema.createMaterializedView('alerts_users_tags', (view) => { - view.columns('user_id', 'tag_id', 'alert_ids'); - - view.as( - knex('alerts_tags') - .select( - 'alerts.user_id', - 'alerts_tags.tag_id', - knex.raw('array_agg(distinct alerts.id) as alert_ids'), - knex.raw('(alerts_actors.id is null and alerts_entities.id is null and alerts_matches.id is null and related_tags.id is null) as is_only'), - ) - .leftJoin('alerts', 'alerts.id', 'alerts_tags.alert_id') - .leftJoin('alerts_entities', 'alerts_entities.alert_id', 'alerts_tags.alert_id') - .leftJoin('alerts_actors', 'alerts_actors.alert_id', 'alerts_tags.alert_id') - .leftJoin('alerts_matches', 'alerts_matches.alert_id', 'alerts_tags.alert_id') - .leftJoin('alerts_tags as related_tags', (joinBuilder) => { - joinBuilder - .on('related_tags.alert_id', 'alerts_tags.alert_id') - .on('related_tags.tag_id', '!=', 'alerts_tags.tag_id'); - }) - .groupBy(['user_id', 'alerts_tags.tag_id', 'is_only']), - ); - }); - - await knex.schema.createMaterializedView('alerts_users_entities', (view) => { - view.columns('user_id', 'entity_id', 'alert_ids'); - - view.as( - knex('alerts_entities') - .select( - 'alerts.user_id', - 'alerts_entities.entity_id', - knex.raw('array_agg(distinct alerts.id) as alert_ids'), - knex.raw('(alerts_actors.id is null and alerts_tags.id is null and alerts_matches.id is null and related_entities.id is null) as is_only'), - ) - .leftJoin('alerts', 'alerts.id', 'alerts_entities.alert_id') - .leftJoin('alerts_tags', 'alerts_tags.alert_id', 'alerts_entities.alert_id') - .leftJoin('alerts_actors', 'alerts_actors.alert_id', 'alerts_entities.alert_id') - .leftJoin('alerts_matches', 'alerts_matches.alert_id', 'alerts_entities.alert_id') - .leftJoin('alerts_entities as related_entities', (joinBuilder) => { - joinBuilder - .on('related_entities.alert_id', 'alerts_entities.alert_id') - .on('related_entities.entity_id', '!=', 'alerts_entities.entity_id'); - }) - .groupBy(['user_id', 'alerts_entities.entity_id', 'is_only']), - ); - }); - - await knex.raw(` - GRANT SELECT ON alerts_users_actors TO :visitor; - GRANT SELECT ON alerts_users_entities TO :visitor; - GRANT SELECT ON alerts_users_tags TO :visitor; - `, { - visitor: knex.raw(config.database.query.user), - }); -}; - -exports.down = async (knex) => { - await knex.schema.alterTable('alerts', (table) => { - table.dropColumn('from_preset'); - table.dropColumn('comment'); - }); - - await knex.schema.dropMaterializedView('alerts_users_actors'); - await knex.schema.dropMaterializedView('alerts_users_tags'); - await knex.schema.dropMaterializedView('alerts_users_entities'); -}; diff --git a/migrations/20250404051102_metadata.js b/migrations/20250404051102_metadata.js deleted file mode 100644 index a61a6ea3..00000000 --- a/migrations/20250404051102_metadata.js +++ /dev/null @@ -1,21 +0,0 @@ -exports.up = async function(knex) { - await knex.schema.alterTable('alerts', (table) => { - table.json('meta'); - }); - - await knex.schema.alterTable('stashes', (table) => { - table.text('comment'); - table.json('meta'); - }); -}; - -exports.down = async function(knex) { - await knex.schema.alterTable('alerts', (table) => { - table.dropColumn('meta'); - }); - - await knex.schema.alterTable('stashes', (table) => { - table.dropColumn('comment'); - table.dropColumn('meta'); - }); -}; diff --git a/migrations/20260122033548_affiliate_parameters.js b/migrations/20260122033548_affiliate_parameters.js deleted file mode 100644 index a90b3763..00000000 --- a/migrations/20260122033548_affiliate_parameters.js +++ /dev/null @@ -1,22 +0,0 @@ -exports.up = async (knex) => { - await knex('affiliates') - .update('parameters', null); - - await knex.schema.alterTable('affiliates', (table) => { - table.dropUnique(['entity_id', 'parameters']); - }); - - await knex.schema.alterTable('affiliates', (table) => { - table.json('parameters') - .alter(); - }); -}; - -exports.down = async (knex) => { - await knex.schema.alterTable('affiliates', (table) => { - table.string('parameters') - .alter(); - - table.unique(['entity_id', 'parameters']); - }); -}; diff --git a/migrations/20260123051908_serie_effective_date.js b/migrations/20260123051908_serie_effective_date.js deleted file mode 100644 index 7708136f..00000000 --- a/migrations/20260123051908_serie_effective_date.js +++ /dev/null @@ -1,14 +0,0 @@ -exports.up = async (knex) => { - await knex.raw(` - /* allow scenes without dates to be mixed inbetween scenes with dates */ - ALTER TABLE series - ADD COLUMN effective_date timestamptz - GENERATED ALWAYS AS (COALESCE(date, created_at)) STORED; - `); -}; - -exports.down = async (knex) => { - await knex.schema.alterTable('series', (table) => { - table.dropColumn('effective_date'); - }); -}; diff --git a/migrations/20260125052234_fingerprints.js b/migrations/20260125052234_fingerprints.js deleted file mode 100644 index 21cfefe3..00000000 --- a/migrations/20260125052234_fingerprints.js +++ /dev/null @@ -1,71 +0,0 @@ -const config = require('config'); - -exports.up = async (knex) => { - await knex.schema.createTable('fingerprints_types', (table) => { - table.string('type') - .primary(); - }); - - await knex('fingerprints_types').insert([ - 'oshash', - 'phash', - 'md5', - 'blake2', - ].map((type) => ({ type }))); - - await knex.schema.createTable('releases_fingerprints', (table) => { - table.increments('id'); - - table.integer('scene_id') - .notNullable() - .references('id') - .inTable('releases'); - - table.string('hash') - .notNullable() - .index(); - - table.string('type') - .notNullable() - .references('type') - .inTable('fingerprints_types'); - - table.integer('duration'); - table.integer('width'); - table.integer('height'); - - table.integer('user_id') - .references('id') - .inTable('users'); - - table.string('source'); - table.integer('source_submissions'); - table.json('source_meta'); - - table.integer('batch_id') - .notNullable() - .references('id') - .inTable('batches'); - - table.datetime('source_created_at'); - - table.datetime('created_at') - .notNullable() - .defaultTo(knex.fn.now()); - }); - - await knex.raw(` - create unique index scenes_fingerprints_unique - on releases_fingerprints (scene_id, hash, source, user_id) - nulls not distinct - `); - - await knex.raw('GRANT ALL ON ALL TABLES IN SCHEMA public TO :visitor;', { - visitor: knex.raw(config.database.query.user), - }); -}; - -exports.down = async function(knex) { - await knex.schema.dropTable('releases_fingerprints'); - await knex.schema.dropTable('fingerprints_types'); -}; diff --git a/migrations/_20230703000556_campaign_html.js b/migrations/_20230703000556_campaign_html.js deleted file mode 100644 index 55fbde8d..00000000 --- a/migrations/_20230703000556_campaign_html.js +++ /dev/null @@ -1,11 +0,0 @@ -exports.up = async (knex) => { - await knex.schema.alterTable('banners', (table) => { - table.text('html'); - }); -}; - -exports.down = async (knex) => { - await knex.schema.alterTable('banners', (table) => { - table.dropColumn('html'); - }); -}; diff --git a/seeds/00_tags.js b/seeds/00_tags.js index 45eb3491..347b0a98 100755 --- a/seeds/00_tags.js +++ b/seeds/00_tags.js @@ -1175,8 +1175,8 @@ const tags = [ slug: 'toys', }, { - name: 'anal toy', - slug: 'anal-toy', + name: 'toy anal', + slug: 'toy-anal', description: 'Stuffing a toy, such as a dildo or buttplug, into the ass', }, { @@ -1701,22 +1701,26 @@ const aliases = [ name: 'brunettes', for: 'brunette', }, + { + name: 'anal toy', + for: 'toy-anal', + }, { name: 'anal toys', - for: 'anal-toy', + for: 'toy-anal', }, { name: 'buttplug', - for: 'anal-toy', + for: 'toy-anal', secondary: true, }, { name: 'butt plug', - for: 'anal-toy', + for: 'toy-anal', }, { name: 'butt plugs', - for: 'anal-toy', + for: 'toy-anal', }, { name: 'caning', @@ -2203,6 +2207,10 @@ const aliases = [ name: 'p.o.v.', for: 'pov', }, + { + name: 'pov sex', + for: 'pov', + }, { name: 'prolapse', for: 'anal-prolapse', @@ -2392,6 +2400,10 @@ const aliases = [ for: 'tattoos', secondary: true, }, + { + name: 'tattoo meid', + for: 'tattoos', + }, { name: 'teens', for: 'teen', @@ -2425,6 +2437,10 @@ const aliases = [ name: 'trans', for: 'transsexual', }, + { + name: 'trans sex', + for: 'transsexual', + }, { name: 'transgender', for: 'transsexual', @@ -2767,6 +2783,14 @@ const aliases = [ name: 'interviews', for: 'interview', }, + { + name: 'grote billen', + for: 'big-butt', + }, + { + name: 'grote lul', + for: 'big-dick', + }, // censors, amateur allure { name: '2 big c---s', @@ -2806,7 +2830,7 @@ const aliases = [ }, { name: 'b--t p--g', - for: 'anal-toy', + for: 'toy-anal', }, { name: 'c--k--g', diff --git a/seeds/02_sites.js b/seeds/02_sites.js index b02b00a1..17b350b4 100755 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -5738,7 +5738,7 @@ const sites = [ name: '5Kporn', url: 'https://www.5kporn.com', tags: ['5k'], - parent: '5kvids', + parent: '8kmembers', parameters: { siteId: 1, short: '5KP', @@ -5749,7 +5749,7 @@ const sites = [ name: '5Kteens', url: 'https://www.5kteens.com', tags: ['5k'], - parent: '5kvids', + parent: '8kmembers', parameters: { siteId: 2, short: '5KT', @@ -5759,7 +5759,7 @@ const sites = [ slug: '8kmilfs', name: '8Kmilfs', url: 'https://www.8kmilfs.com', - parent: '5kvids', + parent: '8kmembers', parameters: { siteId: 3, short: '8KM', @@ -5769,7 +5769,7 @@ const sites = [ slug: '8kteens', name: '8Kteens', url: 'https://www.8kteens.com', - parent: '5kvids', + parent: '8kmembers', parameters: { siteId: 4, short: '8KT', diff --git a/seeds/04_media.js b/seeds/04_media.js index 1acff9df..b41a88a9 100755 --- a/seeds/04_media.js +++ b/seeds/04_media.js @@ -1148,6 +1148,8 @@ exports.seed = (knex) => Promise.resolve() media_id: mediaIdsByPath[photo.path], })); + console.log(tagPosterEntries); + await Promise.all([ upsert('tags_posters', tagPosterEntries, 'tag_id', knex), upsert('tags_photos', tagPhotoEntries, ['tag_id', 'media_id'], knex), diff --git a/seeds/06_affiliates.js b/seeds/06_affiliates.js index 8dc0866a..01794804 100755 --- a/seeds/06_affiliates.js +++ b/seeds/06_affiliates.js @@ -533,7 +533,7 @@ const bannerTags = { slayed_300_250_vicki_violet: ['lesbian', 'brunette', 'pussy-eating'], slayed_315_300_ivy_izzy_103388: ['lesbian', 'blonde', 'brunette', 'ass-eating'], slayed_315_300_vanna_gianna_103313: ['lesbian', 'brunette'], - slayed_728_90_allie_doshi_a: ['lesbian', 'blonde', 'brunette', 'pussy-eating', 'anal-toy'], + slayed_728_90_allie_doshi_a: ['lesbian', 'blonde', 'brunette', 'pussy-eating', 'toy-anal'], slayed_728_90_ariana_alexis_102779: ['lesbian', 'pussy-eating', 'brunette'], slayed_728_90_cecilia_scarlit_102783: ['lesbian', 'black', 'brunette', 'strapon'], slayed_770_76_ivy_izzy_103388: ['lesbian', 'brunette', 'blonde', 'ass-eating'], diff --git a/src/scrapers/bluedonkeymedia.js b/src/scrapers/bluedonkeymedia.js index 1edfb1bc..94827c3b 100644 --- a/src/scrapers/bluedonkeymedia.js +++ b/src/scrapers/bluedonkeymedia.js @@ -1,106 +1,124 @@ 'use strict'; -const crypto = require('crypto'); const unprint = require('unprint'); -const http = require('../utils/http'); +async function fetchTrailer(entryId, videoId, channel, { parameters }) { + // query seems superfluous, but mimics native behavior + const url = `https://apiv2.sysero.nl/api/mvh/stream/get/${entryId}/${videoId}?query=(include:(source:()))`; -async function fetchTrailer(entryId, videoId, channel, credentials) { - const url = `https://api.sysero.nl/free-stream?resource_id=${entryId}&video_id=${videoId}`; - - const res = await http.get(url, { + const res = await unprint.get(url, { headers: { - Origin: channel.url, - Credentials: credentials, + Origin: channel.origin, + Credentials: `sysero ${parameters.credentials}`, }, }); if (res.ok) { - return res.body.data?.attributes.sources.streams.mpd?.url; + return res.data.data?.ITEM?.LINKS?.map((link) => ({ + stream: link.URL, + quality: Number(link.HEIGHT) || null, + })); } return null; } -// MVH's slug system seems to break on non-alphanumerical characters, but also supports ID -function getSceneUrl(channel, slug, sceneId) { - if (slug && /^[\w-]+$/i.test(slug)) { - return `${channel.url}/sexfilms/${slug}`; +async function scrapeSceneData(scene, channel, context, isDeep) { + const release = {}; + + release.entryId = scene.id; + release.url = `${channel.origin}/sexfilms/${scene.slug}`; + + release.title = scene.title; + release.description = scene.description; + + release.date = unprint.extractDate(scene.active_from, 'YYYY-MM-DD HH:mm:ss'); + release.duration = scene.videos?.data?.film?.[0]?.duration; + + release.actors = scene.model?.map((model) => ({ + entryId: model.id, + name: model.title, + url: model.slug && `${channel.origin}/modellen/${model.slug}`, + })); + + release.tags = scene.resources?.data?.filter((tag) => tag.type === 'category').map((tag) => tag.title); + release.language = scene.videos?.data?.film?.[0]?.language; + + if (isDeep) { + const thumb = scene.images?.data?.thumb?.[0]; + + if (thumb) { + release.poster = `https://cdndo.sysero.nl${thumb.path}`; + } + + release.photos = scene.images?.data?.album?.map((image) => `https://cdndo.sysero.nl${image.path}`) || []; + + const videoId = scene.videos?.data?.trailer?.[0]?.id; + + if (context.include.trailers && videoId) { + release.trailer = await fetchTrailer(release.entryId, videoId, channel, context); + } + } else { + [release.poster, ...release.photos] = scene.images?.data?.thumb?.map((image) => `https://cdndo.sysero.nl${image.path}`) || []; } - return `${channel.url}/sexfilms/${sceneId}`; + if (scene.clips?.data?.[0]) { + release.teaser = `https://cdndo.sysero.nl${scene.clips.data[0].path}`; + } + + return release; } function scrapeAll(scenes, channel, context) { - return scenes.reduce((acc, scene) => { - const release = {}; + return scenes.reduce(async (chain, scene) => { + const acc = await chain; - /* - release.entryId = scene.id; - release.url = getSceneUrl(channel, scene.attributes.slug, scene.id); - release.date = unprint.extractDate(scene.attributes.product.active_from, 'D/M/YY'); - - release.title = scene.attributes.title; - release.description = scene.attributes.description; - release.duration = unprint.extractDuration(scene.attributes.videos.film?.[0]?.duration); - - const posterPath = scene.attributes.images.thumb?.[0]?.path || context.images[scene.id]; - const teaserPath = context.clips[scene.relationships.clips?.data[0]?.id]; - - if (posterPath) { - release.poster = `https://cdndo.sysero.nl${scene.attributes.images.thumb?.[0]?.path || context.images[scene.id]}`; + if (!scene.slug) { + // page 2 of Vurig Vlaanderen and possible others return broken entries, even on website + return acc; } - if (teaserPath) { - release.teaser = `https://cdndo.sysero.nl${teaserPath}`; - } - - release.tags = scene.relationships.categories?.data.map((category) => context.tags[category.id]?.replace(/-/g, ' ')).filter(Boolean); - release.language = scene.attributes.videos.film?.[0]?.language; + const release = await scrapeSceneData(scene, channel, context, false); if (release.language && channel.parameters.languages && !channel.parameters.languages?.includes(release.language)) { // all MVH sites list the entire network, but we want to store Flemish scenes under Vurig Vlaanderen + // the international releases should go on MVH, but the API can't filter for NL+EN, so we do it here return { ...acc, unextracted: [...acc.unextracted, release] }; } - */ - - console.log(scene); - console.log(release); return { ...acc, scenes: [...acc.scenes, release] }; - }, { + }, Promise.resolve({ scenes: [], unextracted: [], - }); + })); } async function fetchLatest(channel, page, context) { // query seems to break if any component is left out or even moved - const res = await http.get(`https://apiv2.sysero.nl/api${context.parameters.apiPath}/resources/nl?query=(content:videos,types:(0:video),sort:(published_at:DESC),filters:(active:1,status:published),pagination:(page:${page},per_page:20),include:((resources:(filters:((types:(0:category),status:published)),images:(filters:((types:(0:thumb))))),images:(filters:((types:(0:cover,1:home_cover,2:thumb,3:cover_thumb)))),clips:(),videos:(),categories:())))`, { + // const res = await unprint.get(`https://apiv2.sysero.nl/api${context.parameters.apiPath}/resources/nl?query=(content:videos,types:(0:video),sort:(published_at:DESC),filters:(active:1,status:published),pagination:(page:${page},per_page:20),include:((resources:(filters:((types:(0:category),status:published)),images:(filters:((types:(0:thumb))))),images:(filters:((types:(0:cover,1:home_cover,2:thumb,3:cover_thumb)))),clips:(),videos:(),categories:())))`, { + // const res = await unprint.get(`https://apiv2.sysero.nl/api${context.parameters.apiPath}/resources/nl?query=(content:videos,types:(0:video),sort:(published_at:DESC),filters:(active:1,status:published),pagination:(page:${page},per_page:20),include:((resources:(filters:((types:(0:category),status:published)),images:(filters:((types:(0:thumb))))),images:(filters:((types:(0:cover,1:home_cover,2:thumb,3:cover_thumb)))),clips:(),videos:(),categories:())))`, { + const res = await unprint.get(`https://apiv2.sysero.nl/api${context.parameters.apiPath}/resources/nl?query=(content:videos,types:(0:video),sort:(published_at:DESC),filters:(active:1,status:published),pagination:(page:${page},per_page:20),include:((resources:(filters:((types:(0:category),status:published)),images:(filters:((types:(0:thumb))))),images:(filters:((types:(0:cover,1:home_cover,2:thumb,3:cover_thumb)))),clips:(),videos:(),categories:())))`, { headers: { Origin: channel.origin, - Credentials: context.parameters.credentials, + Credentials: `sysero ${context.parameters.credentials}`, }, }); - if (res.ok && res.body.data) { - return scrapeAll(res.body.data, channel, context); + if (res.ok && res.data.data) { + return scrapeAll(res.data.data, channel, context); } return res.status; } -function getCredentials(channel) { - const now = Math.floor(Date.now() / 1000); +async function scrapeScene({ window }, context) { + const data = window.__NUXT__?.state?.resourcesStore?.video; - const hash = crypto - .createHmac('sha256', channel.parameters.secret) - .update(`${channel.parameters.frontend}${now.toString()}`) - .digest('hex'); + if (data) { + return scrapeSceneData(data, context.entity, context, true); + } - const credentials = `Syserauth ${channel.parameters.frontend}-${hash}-${now.toString(16)}`; - - return credentials; + return null; } const falseCountry = /afghanistan/i; // no country defaults to Afghanistan @@ -114,9 +132,11 @@ function getLocation(model) { .join(', ') || null; } -function scrapeProfile(model, { entity, includeScenes = true }) { +async function scrapeProfile(model, { entity }) { const actor = {}; + // gender unreliable, seems to report everyone as 'vrouw' (woman) + actor.name = model.title; actor.url = unprint.prefixUrl(`/modellen/${model.slug}`, entity.url); @@ -136,76 +156,20 @@ function scrapeProfile(model, { entity, includeScenes = true }) { actor.eyes = model.eye_color; actor.hairColor = model.hair_color; - if (includeScenes) { - actor.scenes = model.videos?.map((video) => ({ - entryId: video.id, - url: getSceneUrl(entity, video.slug, video.id), - title: video.title, - description: video.description, - })); - } - actor.avatar = unprint.prefixUrl(model.images?.[0]?.path, 'https://cdndo.sysero.nl'); + actor.scenes = await Promise.all(model.video?.map(async (video) => scrapeSceneData(video, entity, false))); + return actor; } -function scrapeSceneData(scene, { entity }) { - const release = {}; - - release.entryId = scene.id; - release.url = getSceneUrl(entity, scene.slug, scene.id); - - release.title = scene.title; - release.description = scene.description; - release.date = scene.uploadDate - ? new Date(scene.uploadDate) - : unprint.extractDate(scene.product.active_from, 'D/M/YY'); - - release.actors = scene.models?.map((model) => scrapeProfile(model, { entity, includeScenes: false })); - - release.duration = scene.seconds || unprint.extractTimestamp(scene.isoDuration) || Number(scene.video_paid?.duration) * 60; - release.tags = scene.categories?.map((category) => category.slug.replace(/-/g, ' ')); - - if (scene.thumb) { - release.poster = [ - scene.thumb.original, - scene.thumb.xxl, - scene.thumb.xl, - // ... l, m, s, xs, xxs, probably little point trying all of them - ].map((poster) => unprint.prefixUrl(poster, 'https://cdndo.sysero.nl')); - } - - release.photos = scene.gallery; - - if (scene.trailer) { - release.trailer = async () => { - const credentials = getCredentials(entity); - return fetchTrailer(scene.id, scene.trailer.id, entity, credentials); - }; - } - - return release; -} - -function scrapeScene({ _query, window }, context) { - const data = window.__NUXT__?.state?.videoStore?.video; - - if (data) { - return scrapeSceneData(data, context); - } - - return null; -} - -async function fetchProfile(actor, { entity }) { - const credentials = getCredentials(entity); - const url = `${entity.url}/modellen/${actor.slug}`; +async function fetchProfile(actor, { entity, parameters }) { + const url = actor.url || `${entity.url}/modellen/${actor.slug}`; const res = await unprint.get(url, { headers: { - Origin: entity.url, - Credentials: credentials, + Origin: entity.origin, + Credentials: `sysero ${parameters.credentials}`, }, parser: { runScripts: 'dangerously', diff --git a/src/tags.js b/src/tags.js index 2d8a879e..e930a598 100755 --- a/src/tags.js +++ b/src/tags.js @@ -76,7 +76,7 @@ function withRelations(queryBuilder, withMedia) { async function matchReleaseTags(releases) { const tags = releases .map((release) => release.tags).flat() - .map((tag) => tag?.trim().toLowerCase()) + .map((tag) => tag?.trim().match(/[a-z0-9]+/ig)?.join(' ').toLowerCase()) .filter(Boolean); const tagEntries = await knex('tags')