From a492401db03e1b1b307f8c897d37d59333386184 Mon Sep 17 00:00:00 2001 From: DebaucheryLibrarian Date: Fri, 3 Apr 2026 00:26:13 +0200 Subject: [PATCH] Storing scene language and production date precision. Refactored Teen Core Club. --- ...cene_language_production_date_precision.js | 27 + package-lock.json | 187 +++++++ package.json | 1 + seeds/02_sites.js | 466 +++--------------- seeds/07_languages.js | 196 ++++++++ src/scrapers/teencoreclub.js | 134 +++-- src/store-releases.js | 20 +- 7 files changed, 553 insertions(+), 478 deletions(-) create mode 100644 migrations/20260402051404_scene_language_production_date_precision.js create mode 100755 seeds/07_languages.js diff --git a/migrations/20260402051404_scene_language_production_date_precision.js b/migrations/20260402051404_scene_language_production_date_precision.js new file mode 100644 index 00000000..75ed726b --- /dev/null +++ b/migrations/20260402051404_scene_language_production_date_precision.js @@ -0,0 +1,27 @@ +exports.up = async function(knex) { + await knex.schema.createTable('languages', (table) => { + table.string('alpha2') + .primary(); + + table.text('name'); + table.text('name_native'); + }); + + await knex.schema.alterTable('releases', (table) => { + table.enum('production_date_precision', ['year', 'month', 'week', 'day', 'hour', 'minute', 'second']) + .defaultTo('day'); + + table.string('language_alpha2') + .references('alpha2') + .inTable('languages'); + }); +}; + +exports.down = async function(knex) { + await knex.schema.alterTable('releases', (table) => { + table.dropColumn('production_date_precision'); + table.dropColumn('language_alpha2'); + }); + + await knex.schema.dropTable('languages'); +}; diff --git a/package-lock.json b/package-lock.json index 31496fcf..b9857380 100644 --- a/package-lock.json +++ b/package-lock.json @@ -29,6 +29,7 @@ "casual": "^1.6.2", "cheerio": "^1.0.0-rc.12", "cli-confirm": "^1.0.1", + "clipboardy": "^2.3.0", "cloudscraper": "^4.6.0", "config": "^3.3.9", "connect-session-knex": "^4.0.0", @@ -6005,6 +6006,25 @@ "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", "integrity": "sha512-lYe4Gx7QT+MKGbDsA+Z+he/Wtef0BiwDOlK/XkBrdfsh9J/jPPXbX0tE9x9cl27Tmu5gg3QUbUrQYa/y+KOHPQ==" }, + "node_modules/arch": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/arch/-/arch-2.2.0.tgz", + "integrity": "sha512-Of/R0wqp83cgHozfIYLbBMnej79U/SVGOOyuB3VVFv1NRM/PSFMK12x9KVtiYzJqmnU5WR2qp0Z5rHb7sWGnFQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/are-we-there-yet": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-2.0.0.tgz", @@ -7403,6 +7423,127 @@ "node": ">= 10" } }, + "node_modules/clipboardy": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/clipboardy/-/clipboardy-2.3.0.tgz", + "integrity": "sha512-mKhiIL2DrQIsuXMgBgnfEHOZOryC7kY7YO//TN6c63wlEm3NG5tz+YgY5rVi29KCmq/QQjKYvM7a19+MDOTHOQ==", + "dependencies": { + "arch": "^2.1.1", + "execa": "^1.0.0", + "is-wsl": "^2.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/clipboardy/node_modules/cross-spawn": { + "version": "6.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-6.0.6.tgz", + "integrity": "sha512-VqCUuhcd1iB+dsv8gxPttb5iZh/D0iubSP21g36KXdEuf6I5JiioesUVjpCdHV9MZRUfVFlvwtIUyPfxo5trtw==", + "dependencies": { + "nice-try": "^1.0.4", + "path-key": "^2.0.1", + "semver": "^5.5.0", + "shebang-command": "^1.2.0", + "which": "^1.2.9" + }, + "engines": { + "node": ">=4.8" + } + }, + "node_modules/clipboardy/node_modules/execa": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/execa/-/execa-1.0.0.tgz", + "integrity": "sha512-adbxcyWV46qiHyvSp50TKt05tB4tK3HcmF7/nxfAdhnox83seTDbwnaqKO4sXRy7roHAIFqJP/Rw/AuEbX61LA==", + "dependencies": { + "cross-spawn": "^6.0.0", + "get-stream": "^4.0.0", + "is-stream": "^1.1.0", + "npm-run-path": "^2.0.0", + "p-finally": "^1.0.0", + "signal-exit": "^3.0.0", + "strip-eof": "^1.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/clipboardy/node_modules/get-stream": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-4.1.0.tgz", + "integrity": "sha512-GMat4EJ5161kIy2HevLlr4luNjBgvmj413KaQA7jt4V8B4RDsfpHk7WQ9GVqfYyyx8OS/L66Kox+rJRNklLK7w==", + "dependencies": { + "pump": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/clipboardy/node_modules/is-stream": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-1.1.0.tgz", + "integrity": "sha512-uQPm8kcs47jx38atAcWTVxyltQYoPT68y9aWYdV6yWXSyW8mzSat0TL6CiWdZeCdF3KrAvpVtnHbTv4RN+rqdQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/clipboardy/node_modules/npm-run-path": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-2.0.2.tgz", + "integrity": "sha512-lJxZYlT4DW/bRUtFh1MQIWqmLwQfAxnqWG4HhEdjMlkrJYnJn0Jrr2u3mgxqaWsdiBc76TYkTG/mhrnYTuzfHw==", + "dependencies": { + "path-key": "^2.0.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/clipboardy/node_modules/path-key": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/path-key/-/path-key-2.0.1.tgz", + "integrity": "sha512-fEHGKCSmUSDPv4uoj8AlD+joPlq3peND+HRYyxFz4KPw4z926S/b8rIuFs2FYJg3BwsxJf6A9/3eIdLaYC+9Dw==", + "engines": { + "node": ">=4" + } + }, + "node_modules/clipboardy/node_modules/semver": { + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", + "integrity": "sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==", + "bin": { + "semver": "bin/semver" + } + }, + "node_modules/clipboardy/node_modules/shebang-command": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-1.2.0.tgz", + "integrity": "sha512-EV3L1+UQWGor21OmnvojK36mhg+TyIKDh3iFBKBohr5xeXIhNBcx8oWdgkTEEQ+BEFFYdLRuqMfd5L84N1V5Vg==", + "dependencies": { + "shebang-regex": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/clipboardy/node_modules/shebang-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-1.0.0.tgz", + "integrity": "sha512-wpoSFAxys6b2a2wHZ1XpDSgD7N9iVjg29Ph9uV/uaP9Ex/KXlkTZTeddxDPSYQpgvzKLGJke2UU0AzoGCjNIvQ==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/clipboardy/node_modules/which": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", + "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", + "dependencies": { + "isexe": "^2.0.0" + }, + "bin": { + "which": "bin/which" + } + }, "node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -11513,6 +11654,20 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-docker": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-2.2.1.tgz", + "integrity": "sha512-F+i2BKsFrH66iaUFc0woD8sLy8getkwTwtOBjvs56Cx4CgJDeKQeqfz8wAYiSb8JOprWhHH5p77PbmYCvvUuXQ==", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-extendable": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-1.0.1.tgz", @@ -11821,6 +11976,17 @@ "url": "https://github.com/sponsors/mesqueeb" } }, + "node_modules/is-wsl": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", + "integrity": "sha512-fKzAra0rGJUUBwGBgNkHZuToZcn+TtXHpeCgmkMJMMYx1sQDYaCSyjJBSCa2nH1DGm7s3n1oBnohoVTBaN7Lww==", + "dependencies": { + "is-docker": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/isarray": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", @@ -13629,6 +13795,11 @@ "node": ">= 0.4.0" } }, + "node_modules/nice-try": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/nice-try/-/nice-try-1.0.5.tgz", + "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==" + }, "node_modules/node-domexception": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", @@ -16473,6 +16644,14 @@ "node": ">=0.10.0" } }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "engines": { + "node": ">=4" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -19729,6 +19908,14 @@ "node": ">=4" } }, + "node_modules/strip-eof": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz", + "integrity": "sha512-7FCwGGmx8mD5xQd3RPUvnSpUXHM3BWuzjtpD4TXsfcZ9EL4azvVVUscFYwD9nx8Kh+uCBC00XBtAykoMHwTh8Q==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/strip-final-newline": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", diff --git a/package.json b/package.json index c2047a20..1f167c90 100755 --- a/package.json +++ b/package.json @@ -88,6 +88,7 @@ "casual": "^1.6.2", "cheerio": "^1.0.0-rc.12", "cli-confirm": "^1.0.1", + "clipboardy": "^2.3.0", "cloudscraper": "^4.6.0", "config": "^3.3.9", "connect-session-knex": "^4.0.0", diff --git a/seeds/02_sites.js b/seeds/02_sites.js index a6338eec..aa8f4648 100755 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -13501,13 +13501,7 @@ const sites = [ { name: 'Analyzed Girls', slug: 'analyzedgirls', - url: 'https://analyzedgirls.com', - alias: [ - 'ag', - ], - tags: [ - 'anal', - ], + url: 'https://teencoreclub.com/videos/browse/labels/178', parent: 'teencoreclub', parameters: { legacySiteId: 178, @@ -13516,7 +13510,7 @@ const sites = [ { name: 'Archived Updates', slug: 'archivedupdates', - url: 'https://teencoreclub.com/browsevideos/site/482/ArchivedUpdates', + url: 'https://teencoreclub.com/videos/browse/labels/482', parent: 'teencoreclub', hasLogo: false, parameters: { @@ -13527,30 +13521,15 @@ const sites = [ name: 'Ass Teen Mouth', slug: 'assteenmouth', url: 'https://assteenmouth.com', - alias: [ - 'atm', - ], - tags: [ - 'anal', - 'atm', - 'teen', - ], parent: 'teencoreclub', parameters: { legacySiteId: 180, - siteId: 17, }, }, { name: 'Bang Teen Pussy', slug: 'bangteenpussy', - url: 'https://bangteenpussy.com', - alias: [ - 'btp', - ], - tags: [ - 'teen', - ], + url: 'https://teencoreclub.com/videos/browse/labels/182', parent: 'teencoreclub', parameters: { legacySiteId: 182, @@ -13560,9 +13539,6 @@ const sites = [ name: 'Brutal Invasion', slug: 'brutalinvasion', url: 'https://brutalinvasion.com', - alias: [ - 'bin', - ], parent: 'teencoreclub', parameters: { legacySiteId: 184, @@ -13571,13 +13547,7 @@ const sites = [ { name: 'Clean My Ass', slug: 'cleanmyass', - url: 'https://teencoreclub.com/browsevideos/site/362/CleanMyAss', - tags: [ - 'ass-eating', - ], - alias: [ - 'cma', - ], + url: 'https://teencoreclub.com/videos/browse/labels/362', parent: 'teencoreclub', parameters: { legacySiteId: 362, @@ -13586,10 +13556,7 @@ const sites = [ { name: 'College Party Time', slug: 'collegepartytime', - url: 'https://teencoreclub.com/browsevideos/site/278/CollegePartyTime', - alias: [ - 'cpt', - ], + url: 'https://teencoreclub.com/videos/browse/labels/278', parent: 'teencoreclub', parameters: { legacySiteId: 278, @@ -13598,15 +13565,7 @@ const sites = [ { name: 'Cumaholic Teens', slug: 'cumaholicteens', - url: 'https://cumaholicteens.com', - alias: [ - 'ct', - 'cumoholic teens', - 'cumoholicteens', - ], - tags: [ - 'teen', - ], + url: 'https://teencoreclub.com/videos/browse/labels/186', parent: 'teencoreclub', parameters: { legacySiteId: 186, @@ -13615,10 +13574,7 @@ const sites = [ { name: 'Cum Filled Throat', slug: 'cumfilledthroat', - url: 'https://teencoreclub.com/browsevideos/site/280/CumFilledThroat', - alias: [ - 'cft', - ], + url: 'https://teencoreclub.com/videos/browse/labels/280', parent: 'teencoreclub', parameters: { legacySiteId: 280, @@ -13627,10 +13583,7 @@ const sites = [ { name: 'Cum Sumption Cocktail', slug: 'cumsumptioncocktail', - url: 'https://teencoreclub.com/browsevideos/site/282/CumSumptionCocktail', - alias: [ - 'csc', - ], + url: 'https://teencoreclub.com/videos/browse/labels/282', parent: 'teencoreclub', parameters: { legacySiteId: 282, @@ -13640,9 +13593,6 @@ const sites = [ name: 'Defiled18', slug: 'defiled18', url: 'https://defiled18.com', - alias: [ - 'd18', - ], parent: 'teencoreclub', parameters: { legacySiteId: 188, @@ -13651,10 +13601,7 @@ const sites = [ { name: 'Dirty Baby Sitter', slug: 'dirtybabysitter', - url: 'https://teencoreclub.com/browsevideos/site/284/DirtyBabySitter', - alias: [ - 'dbs', - ], + url: 'https://teencoreclub.com/videos/browse/labels/284', parent: 'teencoreclub', parameters: { legacySiteId: 284, @@ -13664,13 +13611,6 @@ const sites = [ name: 'Double Teamed Teens', slug: 'doubleteamedteens', url: 'https://doubleteamedteens.com', - alias: [ - 'dtt', - ], - tags: [ - 'teen', - 'dp', - ], parent: 'teencoreclub', parameters: { legacySiteId: 190, @@ -13679,15 +13619,7 @@ const sites = [ { name: 'Dreamteens HD', slug: 'dreamteenshd', - url: 'https://dreamteenshd.com', - alias: [ - 'dhd', - 'spearteenpussy', - 'spear teen pussy', - ], - tags: [ - 'teen', - ], + url: 'https://teencoreclub.com/videos/browse/labels/192', parent: 'teencoreclub', parameters: { legacySiteId: 192, @@ -13696,15 +13628,7 @@ const sites = [ { name: 'Drill Her Butt', slug: 'drillherbutt', - url: 'https://teencoreclub.com/browsevideos/site/288/DrillHerButt', - tags: [ - 'anal', - ], - alias: [ - 'dhb', - 'drilledchicks', - 'drilled chicks', - ], + url: 'https://teencoreclub.com/videos/browse/labels/288', parent: 'teencoreclub', parameters: { legacySiteId: 288, @@ -13713,10 +13637,7 @@ const sites = [ { name: 'Dual Throat', slug: 'dualthroat', - url: 'https://teencoreclub.com/browsevideos/site/290/DualThroat', - alias: [ - 'dt', - ], + url: 'https://teencoreclub.com/videos/browse/labels/290', parent: 'teencoreclub', parameters: { legacySiteId: 290, @@ -13725,13 +13646,7 @@ const sites = [ { name: 'Gang Land Victims', slug: 'ganglandvictims', - url: 'https://teencoreclub.com/browsevideos/site/292/GangLandVictims', - tags: [ - 'gangbang', - ], - alias: [ - 'glv', - ], + url: 'https://teencoreclub.com/videos/browse/labels/292', parent: 'teencoreclub', parameters: { legacySiteId: 292, @@ -13741,9 +13656,6 @@ const sites = [ name: 'Girls Got Cream', slug: 'girlsgotcream', url: 'https://girlsgotcream.com', - alias: [ - 'ggc', - ], parent: 'teencoreclub', parameters: { legacySiteId: 194, @@ -13752,10 +13664,7 @@ const sites = [ { name: 'Hardcore Youth', slug: 'hardcoreyouth', - url: 'https://hardcoreyouth.com', - alias: [ - 'hcy', - ], + url: 'https://teencoreclub.com/videos/browse/labels/196', parent: 'teencoreclub', parameters: { legacySiteId: 196, @@ -13764,10 +13673,7 @@ const sites = [ { name: 'Little Hellcat', slug: 'littlehellcat', - url: 'https://littlehellcat.com', - alias: [ - 'lhc', - ], + url: 'https://teencoreclub.com/videos/browse/labels/198', parent: 'teencoreclub', parameters: { legacySiteId: 198, @@ -13776,13 +13682,7 @@ const sites = [ { name: 'Little Teen Suckers', slug: 'littleteensuckers', - url: 'https://teencoreclub.com/browsevideos/site/294/LittleTeenSuckers', - tags: [ - 'teen', - ], - alias: [ - 'lts', - ], + url: 'https://teencoreclub.com/videos/browse/labels/294', parent: 'teencoreclub', parameters: { legacySiteId: 294, @@ -13791,9 +13691,8 @@ const sites = [ { name: 'MAIKO', slug: 'maiko', - url: 'https://teencoreclub.com/browsevideos/site/566/MAIKO', + url: 'https://teencoreclub.com/videos/browse/labels/566', parent: 'teencoreclub', - visible: false, hasLogo: false, parameters: { legacySiteId: 566, @@ -13803,17 +13702,6 @@ const sites = [ name: 'Make Teen Gape', slug: 'maketeengape', url: 'https://maketeengape.com', - alias: [ - 'mtg', - 'analcheckup', - 'analcheckups', - 'anal checkup', - 'anal checkups', - ], - tags: [ - 'teen', - 'gaping', - ], parent: 'teencoreclub', parameters: { legacySiteId: 200, @@ -13822,13 +13710,7 @@ const sites = [ { name: 'Make Teen Moan', slug: 'maketeenmoan', - url: 'https://teencoreclub.com/browsevideos/site/296/MakeTeenMoan', - tags: [ - 'teen', - ], - alias: [ - 'mtm', - ], + url: 'https://teencoreclub.com/videos/browse/labels/296', parent: 'teencoreclub', parameters: { legacySiteId: 296, @@ -13837,10 +13719,7 @@ const sites = [ { name: 'Mega Penetrations', slug: 'megapenetrations', - url: 'https://teencoreclub.com/browsevideos/site/298/MegaPenetrations', - alias: [ - 'mp', - ], + url: 'https://teencoreclub.com/videos/browse/labels/298', parent: 'teencoreclub', parameters: { legacySiteId: 298, @@ -13849,13 +13728,7 @@ const sites = [ { name: 'Messy Gang Bangs', slug: 'messygangbangs', - url: 'https://teencoreclub.com/browsevideos/site/300/MessyGangBangs', - tags: [ - 'gangbang', - ], - alias: [ - 'mgb', - ], + url: 'https://teencoreclub.com/videos/browse/labels/300', parent: 'teencoreclub', parameters: { legacySiteId: 300, @@ -13864,13 +13737,7 @@ const sites = [ { name: 'My Black Coeds', slug: 'myblackcoeds', - url: 'https://teencoreclub.com/browsevideos/site/302/MyBlackCoeds', - tags: [ - 'black', - ], - alias: [ - 'mbc', - ], + url: 'https://teencoreclub.com/videos/browse/labels/302', parent: 'teencoreclub', parameters: { legacySiteId: 302, @@ -13879,14 +13746,7 @@ const sites = [ { name: 'My Latina Teen', slug: 'mylatinateen', - url: 'https://teencoreclub.com/browsevideos/site/304/MyLatinaTeen', - tags: [ - 'teen', - 'latina', - ], - alias: [ - 'mlt', - ], + url: 'https://teencoreclub.com/videos/browse/labels/304', parent: 'teencoreclub', parameters: { legacySiteId: 304, @@ -13895,13 +13755,7 @@ const sites = [ { name: 'Nasty Ass Lickers', slug: 'nastyasslickers', - url: 'https://teencoreclub.com/browsevideos/site/306/NastyAssLickers', - tags: [ - 'ass-eating', - ], - alias: [ - 'nal', - ], + url: 'https://teencoreclub.com/videos/browse/labels/306', parent: 'teencoreclub', parameters: { legacySiteId: 306, @@ -13910,10 +13764,7 @@ const sites = [ { name: 'Naughty Little Nymphs', slug: 'naughtylittlenymphs', - url: 'https://teencoreclub.com/browsevideos/site/308/NaughtyLittleNymphs', - alias: [ - 'nln', - ], + url: 'https://teencoreclub.com/videos/browse/labels/308', parent: 'teencoreclub', parameters: { legacySiteId: 308, @@ -13922,9 +13773,8 @@ const sites = [ { name: 'NC UK Sinners', slug: 'ncuksinners', - url: 'https://teencoreclub.com/browsevideos/site/568/NCUKSinners', + url: 'https://teencoreclub.com/videos/browse/labels/568', parent: 'teencoreclub', - visible: false, hasLogo: false, parameters: { legacySiteId: 568, @@ -13933,9 +13783,8 @@ const sites = [ { name: 'NC Verso Cinema', slug: 'ncversocinema', - url: 'https://teencoreclub.com/browsevideos/site/570/NCVersoCinema', + url: 'https://teencoreclub.com/videos/browse/labels/570', parent: 'teencoreclub', - visible: false, hasLogo: false, parameters: { legacySiteId: 570, @@ -13944,10 +13793,7 @@ const sites = [ { name: 'Network Mixed', slug: 'networkmixed', - url: 'https://teencoreclub.com/browsevideos/site/360/NetworkMixed', - alias: [ - 'anc', - ], + url: 'https://teencoreclub.com/videos/browse/labels/360', parent: 'teencoreclub', hasLogo: false, parameters: { @@ -13957,10 +13803,7 @@ const sites = [ { name: 'Never Done That Before', slug: 'neverdonethatbefore', - url: 'https://teencoreclub.com/browsevideos/site/310/NeverDoneThatBefore', - alias: [ - 'ndt', - ], + url: 'https://teencoreclub.com/videos/browse/labels/310', parent: 'teencoreclub', parameters: { legacySiteId: 310, @@ -13969,12 +13812,7 @@ const sites = [ { name: 'Nylon Sweeties', slug: 'nylonsweeties', - url: 'https://nylonsweeties.com', - alias: [ - 'nsw', - 'nylonspunkjunkies', - 'nylon spunk junkies', - ], + url: 'https://teencoreclub.com/videos/browse/labels/202', parent: 'teencoreclub', parameters: { legacySiteId: 202, @@ -13983,10 +13821,7 @@ const sites = [ { name: 'Pink Eye Sluts', slug: 'pinkeyesluts', - url: 'https://teencoreclub.com/browsevideos/site/312/PinkEyeSluts', - alias: [ - 'pes', - ], + url: 'https://teencoreclub.com/videos/browse/labels/312', parent: 'teencoreclub', parameters: { legacySiteId: 312, @@ -13995,13 +13830,7 @@ const sites = [ { name: 'Plug 2 Holes', slug: 'plug2holes', - url: 'https://teencoreclub.com/browsevideos/site/314/Plug2Holes', - tags: [ - 'dp', - ], - alias: [ - 'p2h', - ], + url: 'https://teencoreclub.com/videos/browse/labels/314', parent: 'teencoreclub', parameters: { legacySiteId: 314, @@ -14010,9 +13839,8 @@ const sites = [ { name: 'Pussy Babes', slug: 'pussybabes', - url: 'https://teencoreclub.com/browsevideos/site/556/PussyBabes', + url: 'https://teencoreclub.com/videos/browse/labels/556', parent: 'teencoreclub', - visible: false, hasLogo: false, parameters: { legacySiteId: 556, @@ -14021,13 +13849,7 @@ const sites = [ { name: 'Road Gang Bangs', slug: 'roadgangbangs', - url: 'https://teencoreclub.com/browsevideos/site/316/RoadGangBangs', - tags: [ - 'gangbang', - ], - alias: [ - 'rgb', - ], + url: 'https://teencoreclub.com/videos/browse/labels/316', parent: 'teencoreclub', parameters: { legacySiteId: 316, @@ -14036,11 +13858,7 @@ const sites = [ { name: 'Russian Teen Updates', slug: 'russianteenupdates', - url: 'https://teencoreclub.com/browsevideos/site/418/RussianTeenUpdates', - tags: [ - 'teen', - 'russian', - ], + url: 'https://teencoreclub.com/videos/browse/labels/418', parent: 'teencoreclub', hasLogo: false, parameters: { @@ -14050,10 +13868,7 @@ const sites = [ { name: 'School Bus Chicks', slug: 'schoolbuschicks', - url: 'https://teencoreclub.com/browsevideos/site/318/SchoolBusChicks', - alias: [ - 'sbc', - ], + url: 'https://teencoreclub.com/videos/browse/labels/318', parent: 'teencoreclub', parameters: { legacySiteId: 318, @@ -14062,10 +13877,7 @@ const sites = [ { name: 'Seductive18', slug: 'seductive18', - url: 'https://seductive18.com', - alias: [ - 's18', - ], + url: 'https://teencoreclub.com/videos/browse/labels/204', parent: 'teencoreclub', parameters: { legacySiteId: 204, @@ -14074,14 +13886,7 @@ const sites = [ { name: 'Show Me Gape', slug: 'showmegape', - url: 'https://teencoreclub.com/browsevideos/site/320/ShowMeGape', - tags: [ - 'anal', - 'gaping', - ], - alias: [ - 'smg', - ], + url: 'https://teencoreclub.com/videos/browse/labels/320', parent: 'teencoreclub', parameters: { legacySiteId: 320, @@ -14090,10 +13895,7 @@ const sites = [ { name: 'Shy Teachers Pet', slug: 'shyteacherspet', - url: 'https://teencoreclub.com/browsevideos/site/322/ShyTeachersPet', - alias: [ - 'stp', - ], + url: 'https://teencoreclub.com/videos/browse/labels/322', parent: 'teencoreclub', parameters: { legacySiteId: 322, @@ -14102,10 +13904,7 @@ const sites = [ { name: 'Small Tits Hunter', slug: 'smalltitshunter', - url: 'https://teencoreclub.com/browsevideos/site/324/SmallTitsHunter', - alias: [ - 'sth', - ], + url: 'https://teencoreclub.com/videos/browse/labels/324', parent: 'teencoreclub', parameters: { legacySiteId: 324, @@ -14114,10 +13913,7 @@ const sites = [ { name: 'Spermantino', slug: 'spermantino', - url: 'https://spermantino.com', - alias: [ - 'spe', - ], + url: 'https://teencoreclub.com/videos/browse/labels/366', parent: 'teencoreclub', parameters: { legacySiteId: 366, @@ -14126,10 +13922,7 @@ const sites = [ { name: 'TCC Network Archives', slug: 'tccnetworkarchives', - url: 'https://teencoreclub.com/browsevideos/site/176/TCCNetworkArchives', - alias: [ - 'all', - ], + url: 'https://teencoreclub.com/videos/browse/labels/176', parent: 'teencoreclub', hasLogo: false, parameters: { @@ -14140,12 +13933,6 @@ const sites = [ name: 'Teach My Ass', slug: 'teachmyass', url: 'https://teachmyass.com', - tags: [ - 'anal', - ], - alias: [ - 'tma', - ], parent: 'teencoreclub', parameters: { legacySiteId: 368, @@ -14154,13 +13941,7 @@ const sites = [ { name: 'Teenagers Going Wild', slug: 'teenagersgoingwild', - url: 'https://teencoreclub.com/browsevideos/site/326/TeenagersGoingWild', - tags: [ - 'teen', - ], - alias: [ - 'tgw', - ], + url: 'https://teencoreclub.com/videos/browse/labels/326', parent: 'teencoreclub', parameters: { legacySiteId: 326, @@ -14170,15 +13951,6 @@ const sites = [ name: 'Teen Anal Casting', slug: 'teenanalcasting', url: 'https://teenanalcasting.com', - alias: [ - 'tac', - 'weneednewtalents', - 'we need new talents', - ], - tags: [ - 'anal', - 'teen', - ], parent: 'teencoreclub', parameters: { legacySiteId: 206, @@ -14187,13 +13959,7 @@ const sites = [ { name: 'Teen Drillers', slug: 'teendrillers', - url: 'https://teendrillers.com', - alias: [ - 'td', - ], - tags: [ - 'teen', - ], + url: 'https://teencoreclub.com/videos/browse/labels/208', parent: 'teencoreclub', parameters: { legacySiteId: 208, @@ -14202,13 +13968,7 @@ const sites = [ { name: 'Teen Gina', slug: 'teengina', - url: 'https://teencoreclub.com/browsevideos/site/210/TeenGina', - tags: [ - 'teen', - ], - alias: [ - 'tg', - ], + url: 'https://teencoreclub.com/videos/browse/labels/210', parent: 'teencoreclub', parameters: { legacySiteId: 210, @@ -14217,15 +13977,7 @@ const sites = [ { name: 'Teens Love Blacks', slug: 'teensloveblacks', - url: 'https://teencoreclub.com/browsevideos/site/328/TeensLoveBlacks', - tags: [ - 'interracial', - 'bbc', - 'teen', - ], - alias: [ - 'tlb', - ], + url: 'https://teencoreclub.com/videos/browse/labels/328', parent: 'teencoreclub', parameters: { legacySiteId: 328, @@ -14234,13 +13986,7 @@ const sites = [ { name: 'Teens Natural Way', slug: 'teensnaturalway', - url: 'https://teensnaturalway.com', - alias: [ - 'tnw', - ], - tags: [ - 'teen', - ], + url: 'https://teencoreclub.com/videos/browse/labels/212', parent: 'teencoreclub', parameters: { legacySiteId: 212, @@ -14249,14 +13995,7 @@ const sites = [ { name: 'Teens Try Anal', slug: 'teenstryanal', - url: 'https://teencoreclub.com/browsevideos/site/330/TeensTryAnal', - tags: [ - 'anal', - 'teen', - ], - alias: [ - 'tta', - ], + url: 'https://teencoreclub.com/videos/browse/labels/330', parent: 'teencoreclub', parameters: { legacySiteId: 330, @@ -14265,15 +14004,7 @@ const sites = [ { name: 'Teens Try Blacks', slug: 'teenstryblacks', - url: 'https://teenstryblacks.com', - alias: [ - 'ttb', - ], - tags: [ - 'interracial', - 'bbc', - 'teen', - ], + url: 'https://teencoreclub.com/videos/browse/labels/214', parent: 'teencoreclub', parameters: { legacySiteId: 214, @@ -14282,14 +14013,7 @@ const sites = [ { name: 'Teens Want Orgies', slug: 'teenswantorgies', - url: 'https://teencoreclub.com/browsevideos/site/332/TeensWantOrgies', - tags: [ - 'teen', - 'threesome', - ], - alias: [ - 'two', - ], + url: 'https://teencoreclub.com/videos/browse/labels/332', parent: 'teencoreclub', parameters: { legacySiteId: 332, @@ -14299,16 +14023,6 @@ const sites = [ name: 'Try Teens', slug: 'tryteens', url: 'https://tryteens.com', - alias: [ - 'tt', - 'fabsluts', - 'fab sluts', - 'teensgoporn', - 'teens go porn', - ], - tags: [ - 'teen', - ], parent: 'teencoreclub', parameters: { legacySiteId: 216, @@ -14317,13 +14031,7 @@ const sites = [ { name: 'Tug Job Queens', slug: 'tugjobqueens', - url: 'https://teencoreclub.com/browsevideos/site/334/TugJobQueens', - tags: [ - 'handjob', - ], - alias: [ - 'tjq', - ], + url: 'https://teencoreclub.com/videos/browse/labels/334', parent: 'teencoreclub', parameters: { legacySiteId: 334, @@ -14332,42 +14040,17 @@ const sites = [ { name: 'UK Sinners', slug: 'uksinners', - url: 'https://teencoreclub.com/browsevideos/site/558/UKSinners', + url: 'https://teencoreclub.com/videos/browse/labels/558', parent: 'teencoreclub', - visible: false, hasLogo: false, parameters: { legacySiteId: 558, }, }, - { - name: 'White Box Black Cocks', - slug: 'whiteboxblackcocks', - url: 'https://teencoreclub.com/browsevideos/site/336/WhiteBoxBlackCocks', - tags: [ - 'interracial', - 'bbc', - ], - alias: [ - 'wbc', - ], - parent: 'teencoreclub', - parameters: { - legacySiteId: 336, - }, - }, { name: 'White Teens Black Cocks', slug: 'whiteteensblackcocks', url: 'https://whiteteensblackcocks.com', - alias: [ - 'wbc', - ], - tags: [ - 'interracial', - 'bbc', - 'teen', - ], parent: 'teencoreclub', parameters: { legacySiteId: 218, @@ -14377,55 +14060,20 @@ const sites = [ name: 'Young Throats', slug: 'youngthroats', url: 'https://youngthroats.com', - tags: [ - 'blowjob', - 'deepthroat', - 'facefucking', - ], - alias: [ - 'yt', - ], parent: 'teencoreclub', parameters: { legacySiteId: 220, }, }, - /* TCC VOD services and unused brands { - name: 'She Got Six', - slug: 'shegotsix', + name: 'White Box Black Cocks', + slug: 'whiteboxblackcocks', + url: 'https://teencoreclub.com/videos/browse/labels/336', parent: 'teencoreclub', - url: 'https://shegotsix.com', - parameters: {}, + parameters: { + legacySiteId: 336, + }, }, - { - name: 'Nightclub', - slug: 'nightclub', - parent: 'teencoreclub', - url: 'https://nightclub.com', - parameters: {}, - }, - { - name: 'Nightclub Live', - slug: 'nightclublive', - parent: 'teencoreclub', - url: 'https://nightclublive.com', - parameters: {}, - }, - { - name: "Let's Go Dirty", - slug: 'letsgodirty', - parent: 'teencoreclub', - url: 'https://letsgodirty.com', - }, - { - name: 'Jerk-Off Pass', - slug: 'jerkoffpass', - parent: 'teencoreclub', - url: 'https://jerkoffpass.com', - parameters: {}, - }, - */ // TEEN MEGA WORLD { name: 'TMW VR Net', diff --git a/seeds/07_languages.js b/seeds/07_languages.js new file mode 100755 index 00000000..2422f27d --- /dev/null +++ b/seeds/07_languages.js @@ -0,0 +1,196 @@ +const upsert = require('../src/utils/upsert'); + +const languages = { + ab: { name: 'Abkhaz', nativeName: 'аҧсуа' }, + aa: { name: 'Afar', nativeName: 'Afaraf' }, + af: { name: 'Afrikaans', nativeName: 'Afrikaans' }, + ak: { name: 'Akan', nativeName: 'Akan' }, + sq: { name: 'Albanian', nativeName: 'Shqip' }, + am: { name: 'Amharic', nativeName: 'አማርኛ' }, + ar: { name: 'Arabic', nativeName: 'العربية' }, + an: { name: 'Aragonese', nativeName: 'Aragonés' }, + hy: { name: 'Armenian', nativeName: 'Հայերեն' }, + as: { name: 'Assamese', nativeName: 'অসমীয়া' }, + av: { name: 'Avaric', nativeName: 'авар мацӀ, магӀарул мацӀ' }, + ae: { name: 'Avestan', nativeName: 'avesta' }, + ay: { name: 'Aymara', nativeName: 'aymar aru' }, + az: { name: 'Azerbaijani', nativeName: 'azərbaycan dili' }, + bm: { name: 'Bambara', nativeName: 'bamanankan' }, + ba: { name: 'Bashkir', nativeName: 'башҡорт теле' }, + eu: { name: 'Basque', nativeName: 'euskara, euskera' }, + be: { name: 'Belarusian', nativeName: 'Беларуская' }, + bn: { name: 'Bengali', nativeName: 'বাংলা' }, + bh: { name: 'Bihari', nativeName: 'भोजपुरी' }, + bi: { name: 'Bislama', nativeName: 'Bislama' }, + bs: { name: 'Bosnian', nativeName: 'bosanski jezik' }, + br: { name: 'Breton', nativeName: 'brezhoneg' }, + bg: { name: 'Bulgarian', nativeName: 'български език' }, + my: { name: 'Burmese', nativeName: 'ဗမာစာ' }, + ca: { name: 'Catalan; Valencian', nativeName: 'Català' }, + ch: { name: 'Chamorro', nativeName: 'Chamoru' }, + ce: { name: 'Chechen', nativeName: 'нохчийн мотт' }, + ny: { name: 'Chichewa; Chewa; Nyanja', nativeName: 'chiCheŵa, chinyanja' }, + zh: { name: 'Chinese', nativeName: '中文 (Zhōngwén), 汉语, 漢語' }, + cv: { name: 'Chuvash', nativeName: 'чӑваш чӗлхи' }, + kw: { name: 'Cornish', nativeName: 'Kernewek' }, + co: { name: 'Corsican', nativeName: 'corsu, lingua corsa' }, + cr: { name: 'Cree', nativeName: 'ᓀᐦᐃᔭᐍᐏᐣ' }, + hr: { name: 'Croatian', nativeName: 'hrvatski' }, + cs: { name: 'Czech', nativeName: 'česky, čeština' }, + da: { name: 'Danish', nativeName: 'dansk' }, + dv: { name: 'Divehi; Dhivehi; Maldivian;', nativeName: 'ދިވެހި' }, + nl: { name: 'Dutch', nativeName: 'Nederlands, Vlaams' }, + en: { name: 'English', nativeName: 'English' }, + eo: { name: 'Esperanto', nativeName: 'Esperanto' }, + et: { name: 'Estonian', nativeName: 'eesti, eesti keel' }, + ee: { name: 'Ewe', nativeName: 'Eʋegbe' }, + fo: { name: 'Faroese', nativeName: 'føroyskt' }, + fj: { name: 'Fijian', nativeName: 'vosa Vakaviti' }, + fi: { name: 'Finnish', nativeName: 'suomi, suomen kieli' }, + fr: { name: 'French', nativeName: 'français, langue française' }, + ff: { name: 'Fula; Fulah; Pulaar; Pular', nativeName: 'Fulfulde, Pulaar, Pular' }, + gl: { name: 'Galician', nativeName: 'Galego' }, + ka: { name: 'Georgian', nativeName: 'ქართული' }, + de: { name: 'German', nativeName: 'Deutsch' }, + el: { name: 'Greek, Modern', nativeName: 'Ελληνικά' }, + gn: { name: 'Guaraní', nativeName: 'Avañeẽ' }, + gu: { name: 'Gujarati', nativeName: 'ગુજરાતી' }, + ht: { name: 'Haitian; Haitian Creole', nativeName: 'Kreyòl ayisyen' }, + ha: { name: 'Hausa', nativeName: 'Hausa, هَوُسَ' }, + he: { name: 'Hebrew (modern)', nativeName: 'עברית' }, + hz: { name: 'Herero', nativeName: 'Otjiherero' }, + hi: { name: 'Hindi', nativeName: 'हिन्दी, हिंदी' }, + ho: { name: 'Hiri Motu', nativeName: 'Hiri Motu' }, + hu: { name: 'Hungarian', nativeName: 'Magyar' }, + ia: { name: 'Interlingua', nativeName: 'Interlingua' }, + id: { name: 'Indonesian', nativeName: 'Bahasa Indonesia' }, + ie: { name: 'Interlingue', nativeName: 'Originally called Occidental; then Interlingue after WWII' }, + ga: { name: 'Irish', nativeName: 'Gaeilge' }, + ig: { name: 'Igbo', nativeName: 'Asụsụ Igbo' }, + ik: { name: 'Inupiaq', nativeName: 'Iñupiaq, Iñupiatun' }, + io: { name: 'Ido', nativeName: 'Ido' }, + is: { name: 'Icelandic', nativeName: 'Íslenska' }, + it: { name: 'Italian', nativeName: 'Italiano' }, + iu: { name: 'Inuktitut', nativeName: 'ᐃᓄᒃᑎᑐᑦ' }, + ja: { name: 'Japanese', nativeName: '日本語 (にほんご／にっぽんご)' }, + jv: { name: 'Javanese', nativeName: 'basa Jawa' }, + kl: { name: 'Kalaallisut, Greenlandic', nativeName: 'kalaallisut, kalaallit oqaasii' }, + kn: { name: 'Kannada', nativeName: 'ಕನ್ನಡ' }, + kr: { name: 'Kanuri', nativeName: 'Kanuri' }, + ks: { name: 'Kashmiri', nativeName: 'कश्मीरी, كشميري‎' }, + kk: { name: 'Kazakh', nativeName: 'Қазақ тілі' }, + km: { name: 'Khmer', nativeName: 'ភាសាខ្មែរ' }, + ki: { name: 'Kikuyu, Gikuyu', nativeName: 'Gĩkũyũ' }, + rw: { name: 'Kinyarwanda', nativeName: 'Ikinyarwanda' }, + ky: { name: 'Kirghiz, Kyrgyz', nativeName: 'кыргыз тили' }, + kv: { name: 'Komi', nativeName: 'коми кыв' }, + kg: { name: 'Kongo', nativeName: 'KiKongo' }, + ko: { name: 'Korean', nativeName: '한국어 (韓國語), 조선말 (朝鮮語)' }, + ku: { name: 'Kurdish', nativeName: 'Kurdî, كوردی‎' }, + kj: { name: 'Kwanyama, Kuanyama', nativeName: 'Kuanyama' }, + la: { name: 'Latin', nativeName: 'latine, lingua latina' }, + lb: { name: 'Luxembourgish, Letzeburgesch', nativeName: 'Lëtzebuergesch' }, + lg: { name: 'Luganda', nativeName: 'Luganda' }, + li: { name: 'Limburgish, Limburgan, Limburger', nativeName: 'Limburgs' }, + ln: { name: 'Lingala', nativeName: 'Lingála' }, + lo: { name: 'Lao', nativeName: 'ພາສາລາວ' }, + lt: { name: 'Lithuanian', nativeName: 'lietuvių kalba' }, + lu: { name: 'Luba-Katanga', nativeName: '' }, + lv: { name: 'Latvian', nativeName: 'latviešu valoda' }, + gv: { name: 'Manx', nativeName: 'Gaelg, Gailck' }, + mk: { name: 'Macedonian', nativeName: 'македонски јазик' }, + mg: { name: 'Malagasy', nativeName: 'Malagasy fiteny' }, + ms: { name: 'Malay', nativeName: 'bahasa Melayu, بهاس ملايو‎' }, + ml: { name: 'Malayalam', nativeName: 'മലയാളം' }, + mt: { name: 'Maltese', nativeName: 'Malti' }, + mi: { name: 'Māori', nativeName: 'te reo Māori' }, + mr: { name: 'Marathi (Marāṭhī)', nativeName: 'मराठी' }, + mh: { name: 'Marshallese', nativeName: 'Kajin M̧ajeļ' }, + mn: { name: 'Mongolian', nativeName: 'монгол' }, + na: { name: 'Nauru', nativeName: 'Ekakairũ Naoero' }, + nv: { name: 'Navajo, Navaho', nativeName: 'Diné bizaad, Dinékʼehǰí' }, + nb: { name: 'Norwegian Bokmål', nativeName: 'Norsk bokmål' }, + nd: { name: 'North Ndebele', nativeName: 'isiNdebele' }, + ne: { name: 'Nepali', nativeName: 'नेपाली' }, + ng: { name: 'Ndonga', nativeName: 'Owambo' }, + nn: { name: 'Norwegian Nynorsk', nativeName: 'Norsk nynorsk' }, + no: { name: 'Norwegian', nativeName: 'Norsk' }, + ii: { name: 'Nuosu', nativeName: 'ꆈꌠ꒿ Nuosuhxop' }, + nr: { name: 'South Ndebele', nativeName: 'isiNdebele' }, + oc: { name: 'Occitan', nativeName: 'Occitan' }, + oj: { name: 'Ojibwe, Ojibwa', nativeName: 'ᐊᓂᔑᓈᐯᒧᐎᓐ' }, + cu: { name: 'Old Church Slavonic, Church Slavic, Church Slavonic, Old Bulgarian, Old Slavonic', nativeName: 'ѩзыкъ словѣньскъ' }, + om: { name: 'Oromo', nativeName: 'Afaan Oromoo' }, + or: { name: 'Oriya', nativeName: 'ଓଡ଼ିଆ' }, + os: { name: 'Ossetian, Ossetic', nativeName: 'ирон æвзаг' }, + pa: { name: 'Panjabi, Punjabi', nativeName: 'ਪੰਜਾਬੀ, پنجابی‎' }, + pi: { name: 'Pāli', nativeName: 'पाऴि' }, + fa: { name: 'Persian', nativeName: 'فارسی' }, + pl: { name: 'Polish', nativeName: 'polski' }, + ps: { name: 'Pashto, Pushto', nativeName: 'پښتو' }, + pt: { name: 'Portuguese', nativeName: 'Português' }, + qu: { name: 'Quechua', nativeName: 'Runa Simi, Kichwa' }, + rm: { name: 'Romansh', nativeName: 'rumantsch grischun' }, + rn: { name: 'Kirundi', nativeName: 'kiRundi' }, + ro: { name: 'Romanian, Moldavian, Moldovan', nativeName: 'română' }, + ru: { name: 'Russian', nativeName: 'русский язык' }, + sa: { name: 'Sanskrit (Saṁskṛta)', nativeName: 'संस्कृतम्' }, + sc: { name: 'Sardinian', nativeName: 'sardu' }, + sd: { name: 'Sindhi', nativeName: 'सिन्धी, سنڌي، سندھی‎' }, + se: { name: 'Northern Sami', nativeName: 'Davvisámegiella' }, + sm: { name: 'Samoan', nativeName: 'gagana faa Samoa' }, + sg: { name: 'Sango', nativeName: 'yângâ tî sängö' }, + sr: { name: 'Serbian', nativeName: 'српски језик' }, + gd: { name: 'Scottish Gaelic; Gaelic', nativeName: 'Gàidhlig' }, + sn: { name: 'Shona', nativeName: 'chiShona' }, + si: { name: 'Sinhala, Sinhalese', nativeName: 'සිංහල' }, + sk: { name: 'Slovak', nativeName: 'slovenčina' }, + sl: { name: 'Slovene', nativeName: 'slovenščina' }, + so: { name: 'Somali', nativeName: 'Soomaaliga, af Soomaali' }, + st: { name: 'Southern Sotho', nativeName: 'Sesotho' }, + es: { name: 'Spanish; Castilian', nativeName: 'español, castellano' }, + su: { name: 'Sundanese', nativeName: 'Basa Sunda' }, + sw: { name: 'Swahili', nativeName: 'Kiswahili' }, + ss: { name: 'Swati', nativeName: 'SiSwati' }, + sv: { name: 'Swedish', nativeName: 'svenska' }, + ta: { name: 'Tamil', nativeName: 'தமிழ்' }, + te: { name: 'Telugu', nativeName: 'తెలుగు' }, + tg: { name: 'Tajik', nativeName: 'тоҷикӣ, toğikī, تاجیکی‎' }, + th: { name: 'Thai', nativeName: 'ไทย' }, + ti: { name: 'Tigrinya', nativeName: 'ትግርኛ' }, + bo: { name: 'Tibetan Standard, Tibetan, Central', nativeName: 'བོད་ཡིག' }, + tk: { name: 'Turkmen', nativeName: 'Türkmen, Түркмен' }, + tl: { name: 'Tagalog', nativeName: 'Wikang Tagalog, ᜏᜒᜃᜅ᜔ ᜆᜄᜎᜓᜄ᜔' }, + tn: { name: 'Tswana', nativeName: 'Setswana' }, + to: { name: 'Tonga (Tonga Islands)', nativeName: 'faka Tonga' }, + tr: { name: 'Turkish', nativeName: 'Türkçe' }, + ts: { name: 'Tsonga', nativeName: 'Xitsonga' }, + tt: { name: 'Tatar', nativeName: 'татарча, tatarça, تاتارچا‎' }, + tw: { name: 'Twi', nativeName: 'Twi' }, + ty: { name: 'Tahitian', nativeName: 'Reo Tahiti' }, + ug: { name: 'Uighur, Uyghur', nativeName: 'Uyƣurqə, ئۇيغۇرچە‎' }, + uk: { name: 'Ukrainian', nativeName: 'українська' }, + ur: { name: 'Urdu', nativeName: 'اردو' }, + uz: { name: 'Uzbek', nativeName: 'zbek, Ўзбек, أۇزبېك‎' }, + ve: { name: 'Venda', nativeName: 'Tshivenḓa' }, + vi: { name: 'Vietnamese', nativeName: 'Tiếng Việt' }, + vo: { name: 'Volapük', nativeName: 'Volapük' }, + wa: { name: 'Walloon', nativeName: 'Walon' }, + cy: { name: 'Welsh', nativeName: 'Cymraeg' }, + wo: { name: 'Wolof', nativeName: 'Wollof' }, + fy: { name: 'Western Frisian', nativeName: 'Frysk' }, + xh: { name: 'Xhosa', nativeName: 'isiXhosa' }, + yi: { name: 'Yiddish', nativeName: 'ייִדיש' }, + yo: { name: 'Yoruba', nativeName: 'Yorùbá' }, + za: { name: 'Zhuang, Chuang', nativeName: 'Saɯ cueŋƅ, Saw cuengh' }, +}; + +exports.seed = async (knex) => { + const curatedLanguages = Object.entries(languages).map(([alpha2, names]) => ({ + alpha2: alpha2.toUpperCase(), + name: names.name, + name_native: names.nativeName, + })); + + await upsert('languages', curatedLanguages, 'alpha2', knex); +}; diff --git a/src/scrapers/teencoreclub.js b/src/scrapers/teencoreclub.js index 217bf0f8..1a9e9c3b 100755 --- a/src/scrapers/teencoreclub.js +++ b/src/scrapers/teencoreclub.js @@ -2,83 +2,82 @@ const unprint = require('unprint'); -const slugify = require('../utils/slugify'); +function pickLocale(item) { + if (!item) { + return null; + } -function scrapeAll(scenes) { - return scenes.map(({ query }) => { - const release = {}; + if (item.en) { + return item.en; + } - release.url = query.url('.title a'); - release.entryId = new URL(release.url).pathname.match(/\/scene\/(\d+)/)[1]; - - release.title = query.content('.title a'); - - release.date = query.date('.date', 'MMM DD, YYYY'); - release.duration = query.duration('.duration'); - - release.actors = query.all('.models a.model').map((actorEl) => ({ - name: unprint.query.content(actorEl), - url: unprint.query.url(actorEl, null), - })); - - release.poster = query.img('img.poster'); - release.teaser = query.video('.teaser video'); - - console.log(release); - - return release; - }); + return Object.values(item)[0]; } -async function fetchLatest(channel, page = 1) { - const url = `${channel.url}/${page}`; - const res = await unprint.get(url, { selectAll: '.scene' }); +function scrapeScene(scene, channel) { + const release = {}; - if (res.ok) { - return scrapeAll(res.context, channel); + release.entryId = scene.id; + release.url = `${channel.url}/video/${scene.id}/${scene.slug}`; + + release.title = pickLocale(scene.title); + release.description = pickLocale(scene.description); + + release.date = new Date(scene.publication_date); + release.duration = scene.meta?.duration_seconds || unprint.extractDuration(scene.meta?.duration); + + release.productionDate = scene.meta.year && new Date(Date.UTC(scene.meta.year, 0, 1)); + release.productionDatePrecision = 'year'; + + release.actors = scene.actors?.map((actor) => ({ + name: actor.name, + entryId: actor.id, + url: `${channel.url}/videos/browse/cast/${actor.id}`, + })); + + const poster = scene.artwork?.original; + const photos = [scene.artwork_f16, scene.cover].map((art) => art.original).filter(Boolean); + + if (poster) { + release.poster = poster; + release.photos = photos; + } else { + // not observed, but artwork_f16 is suitable as poster + release.poster = photos[0]; + release.photos = photos.slice(1); + } + + release.caps = scene.screenshots?.map((src) => unprint.prefixUrl(src, 'https://s02.uni73d.net')) || []; + release.teaser = unprint.prefixUrl(scene.preview?.url, 'https://s02.uni73d.net'); + + release.tags = scene.display_genres?.map((genre) => pickLocale(genre.title)).filter(Boolean) || []; + + if (scene.is_gay) { + release.tags = release.tags.concat('gay'); + } + + release.language = scene.meta?.language; + + return release; +} + +async function fetchLatest(channel, page = 1, { parameters }) { + const url = `https://api.fundorado.com/api/videos/browse/labels/${parameters.legacySiteId}?page=${page}&sg=false&sort=release&video_type=scene&lang=en`; + const res = await unprint.get(url); + + if (res.ok && res.data?.videos?.data) { + return res.data.videos.data.map((scene) => scrapeScene(scene, channel)); } return res.status; } -function scrapeScene({ query }, { url }) { - const release = {}; +async function fetchScene(url, channel) { + const entryId = new URL(url).pathname.match(/\/video\/(\d+)/)[1]; + const res = await unprint.get(`https://api.fundorado.com/api/videodetail/${entryId}`); - release.entryId = new URL(url).pathname.match(/\/scene\/(\d+)/)[1]; - - release.title = query.content('h3.title'); - release.description = query.content('p.description'); - - release.date = query.date('.date', 'MMMM D, YYYY'); - release.duration = query.duration('.duration'); - - [release.poster, ...release.photos] = query.imgs('.preview-thumb'); - release.trailer = query.video('.trailer video'); - - console.log(release); - - return release; -} - -function scrapeProfile({ query }) { - const profile = {}; - - profile.description = query.content('.bio-text'); - profile.birthPlace = query.content('.birth-place span'); - - profile.avatar = query.img('.actor-photo img'); - - console.log(profile); - - return profile; -} - -async function fetchProfile({ name: actorName }, entity) { - const url = `${entity.url}/actors/${slugify(actorName, '_')}`; - const res = await unprint.get(url); - - if (res.ok) { - return scrapeProfile(res.context, entity); + if (res.ok && res.data?.video) { + return scrapeScene(res.data.video, channel); } return res.status; @@ -86,6 +85,5 @@ async function fetchProfile({ name: actorName }, entity) { module.exports = { fetchLatest, - fetchProfile, - scrapeScene, + fetchScene, }; diff --git a/src/store-releases.js b/src/store-releases.js index 5eec5788..a2141a2d 100755 --- a/src/store-releases.js +++ b/src/store-releases.js @@ -70,6 +70,7 @@ async function curateReleaseEntry(release, batchId, existingRelease, type = 'sce if (type === 'scene') { curatedRelease.shoot_id = release.shootId || null; curatedRelease.production_date = Number(release.productionDate) ? release.productionDate : null; + curatedRelease.production_date_precision = release.productionDatePrecision; curatedRelease.duration = Math.round(release.duration) || null; // float may happen if scraper converts duration from milliseconds with a simple / 1000 curatedRelease.qualities = Array.from(new Set(release.qualities?.map(Number).filter(Boolean))).sort((qualityA, qualityB) => qualityB - qualityA); } @@ -89,6 +90,20 @@ async function curateReleaseEntry(release, batchId, existingRelease, type = 'sce } } + if (release.language) { + const curatedLanguage = release.language.toLowerCase(); + + const language = await knex('languages') + .where(knex.raw('lower(alpha2)'), curatedLanguage) + .orWhere(knex.raw('lower(name)'), curatedLanguage) + .orWhere(knex.raw('lower(name_native)'), curatedLanguage) + .first(); + + if (language) { + curatedRelease.language_alpha2 = language.alpha2; + } + } + if (!existingRelease && !release.id) { curatedRelease.created_batch_id = batchId; } @@ -443,12 +458,15 @@ async function storeScenes(releases, useBatchId) { description = COALESCE(new.description, releases.description), shoot_id = COALESCE(new.shoot_id, releases.shoot_id), duration = COALESCE(new.duration, releases.duration), + production_date = COALESCE(new.production_date, releases.production_date), + production_date_precision = COALESCE(new.production_date_precision, releases.production_date_precision), + language_alpha2 = COALESCE(new.language_alpha2, releases.language_alpha2), comment = COALESCE(new.comment, releases.comment), attributes = COALESCE(new.attributes::jsonb || releases.attributes::jsonb, new.attributes::jsonb, releases.attributes::jsonb), deep = new.url IS NOT NULL, updated_at = NOW() FROM json_to_recordset(:scenes) - AS new(id int, url text, date timestamptz, entity json, title text, description text, shoot_id text, duration integer, comment text, attributes json, deep boolean) + AS new(id int, url text, date timestamptz, entity json, title text, description text, shoot_id text, duration integer, production_date timestamptz, production_date_precision text, language_alpha2 text, comment text, attributes json, deep boolean) WHERE releases.id = new.id RETURNING releases.* `, {