diff --git a/assets/components/tags/tags.vue b/assets/components/tags/tags.vue index 397b7aa8..36ea547d 100644 --- a/assets/components/tags/tags.vue +++ b/assets/components/tags/tags.vue @@ -52,6 +52,7 @@ async function mounted() { 'ebony', 'latina', 'caucasian', + 'indian', 'natural-boobs', 'fake-boobs', 'blonde', diff --git a/package-lock.json b/package-lock.json index 46ed6ef2..25664f02 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4614,6 +4614,15 @@ "integrity": "sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg==", "dev": true }, + "fluent-ffmpeg": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz", + "integrity": "sha1-yVLeIkD4EuvaCqgAbXd27irPfXQ=", + "requires": { + "async": ">=0.2.9", + "which": "^1.1.1" + } + }, "flush-write-stream": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-1.1.1.tgz", @@ -5721,28 +5730,12 @@ "minimalistic-assert": "^1.0.1" } }, - "hashish": { - "version": "0.0.4", - "resolved": "https://registry.npmjs.org/hashish/-/hashish-0.0.4.tgz", - "integrity": "sha1-bWC8b/r3Ebav1g5CbQd5iAFOZVQ=", - "requires": { - "traverse": ">=0.2.4" - } - }, "he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", "dev": true }, - "hh-mm-ss": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/hh-mm-ss/-/hh-mm-ss-1.2.0.tgz", - "integrity": "sha512-f4I9Hz1dLpX/3mrEs7yq30+FiuO3tt5NWAqAGeBTaoeoBfB8vhcQ3BphuDc5DjZb/K809agqrAaFlP0jhEU/8w==", - "requires": { - "zero-fill": "^2.2.3" - } - }, "hmac-drbg": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz", @@ -5845,11 +5838,6 @@ "debug": "^3.1.0" } }, - "human-signals": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-1.1.1.tgz", - "integrity": "sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw==" - }, "iconv-lite": { "version": "0.5.1", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.1.tgz", @@ -7168,11 +7156,6 @@ } } }, - "merge-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", - "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==" - }, "mersenne-twister": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/mersenne-twister/-/mersenne-twister-1.1.0.tgz", @@ -10456,14 +10439,6 @@ "mixme": "^0.3.1" } }, - "streamify": { - "version": "0.2.9", - "resolved": "https://registry.npmjs.org/streamify/-/streamify-0.2.9.tgz", - "integrity": "sha512-8pUxeLEef9UO1FxtTt5iikAiyzGI4SZRnGuJ3sz8axZ5Xk+/7ezEV5kuJQsMEFxw7AKYw3xp0Ow+20mmSaJbQQ==", - "requires": { - "hashish": "~0.0.4" - } - }, "strict-uri-encode": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz", @@ -10542,11 +10517,6 @@ "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=", "dev": true }, - "strip-final-newline": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", - "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==" - }, "strip-indent": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-1.0.1.tgz", @@ -10966,11 +10936,6 @@ "punycode": "^2.1.1" } }, - "traverse": { - "version": "0.6.6", - "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.6.6.tgz", - "integrity": "sha1-y99WD9e5r2MlAv7UD5GMFX6pcTc=" - }, "trim-newlines": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz", @@ -12110,127 +12075,6 @@ "camelcase": "^5.0.0", "decamelize": "^1.2.0" } - }, - "youtube-dl": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/youtube-dl/-/youtube-dl-3.0.2.tgz", - "integrity": "sha512-LFFfpsYbRLpqKsnb4gzbnyN7fm190tJw3gJVSvfoEfnb/xYIPNT6i9G3jdzPDp/U5cwB3OSq63nUa7rUwxXAGA==", - "requires": { - "debug": "~4.1.1", - "execa": "~3.2.0", - "hh-mm-ss": "~1.2.0", - "mkdirp": "~0.5.1", - "request": "~2.88.0", - "streamify": "~0.2.9", - "universalify": "~0.1.2" - }, - "dependencies": { - "cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "requires": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - } - }, - "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", - "requires": { - "ms": "^2.1.1" - } - }, - "execa": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/execa/-/execa-3.2.0.tgz", - "integrity": "sha512-kJJfVbI/lZE1PZYDI5VPxp8zXPO9rtxOkhpZ0jMKha56AI9y2gGVC6bkukStQf0ka5Rh15BA5m7cCCH4jmHqkw==", - "requires": { - "cross-spawn": "^7.0.0", - "get-stream": "^5.0.0", - "human-signals": "^1.1.1", - "is-stream": "^2.0.0", - "merge-stream": "^2.0.0", - "npm-run-path": "^4.0.0", - "onetime": "^5.1.0", - "p-finally": "^2.0.0", - "signal-exit": "^3.0.2", - "strip-final-newline": "^2.0.0" - } - }, - "get-stream": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz", - "integrity": "sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==", - "requires": { - "pump": "^3.0.0" - } - }, - "is-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz", - "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==" - }, - "mimic-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==" - }, - "npm-run-path": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", - "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", - "requires": { - "path-key": "^3.0.0" - } - }, - "onetime": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.0.tgz", - "integrity": "sha512-5NcSkPHhwTVFIQN+TUqXoS5+dlElHXdpAWu9I0HP20YOtIi+aZ0Ct82jdlILDxjLEAWwvm+qj1m6aEtsDVmm6Q==", - "requires": { - "mimic-fn": "^2.1.0" - } - }, - "p-finally": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-2.0.1.tgz", - "integrity": "sha512-vpm09aKwq6H9phqRQzecoDpD8TmVyGw70qmWlyq5onxY7tqyTTFVvxMykxQSQKILBSFlbXpypIw2T1Ml7+DDtw==" - }, - "path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" - }, - "shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "requires": { - "shebang-regex": "^3.0.0" - } - }, - "shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" - }, - "which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "requires": { - "isexe": "^2.0.0" - } - } - } - }, - "zero-fill": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/zero-fill/-/zero-fill-2.2.3.tgz", - "integrity": "sha1-o97wa6XjmuZEhQu0yirUEStIVek=" } } } diff --git a/package.json b/package.json index 4fe4c23c..6df3a0d7 100644 --- a/package.json +++ b/package.json @@ -91,6 +91,7 @@ "express-react-views": "^0.11.0", "face-api.js": "^0.21.0", "file-type": "^14.1.4", + "fluent-ffmpeg": "^2.1.2", "fs-extra": "^7.0.1", "graphile-utils": "^4.5.6", "iconv-lite": "^0.5.1", @@ -124,7 +125,6 @@ "vuex": "^3.1.2", "winston": "^3.2.1", "winston-daily-rotate-file": "^4.4.2", - "yargs": "^13.3.0", - "youtube-dl": "^3.0.2" + "yargs": "^13.3.0" } } diff --git a/public/img/logos/elegantangel/favicon.png b/public/img/logos/elegantangel/favicon.png new file mode 100644 index 00000000..b06952a0 Binary files /dev/null and b/public/img/logos/elegantangel/favicon.png differ diff --git a/public/img/logos/elegantangel/favicon/favicon_dark.png b/public/img/logos/elegantangel/favicon/favicon_dark.png new file mode 100644 index 00000000..7ebaaf6f Binary files /dev/null and b/public/img/logos/elegantangel/favicon/favicon_dark.png differ diff --git a/public/img/tags/blonde/0.jpeg b/public/img/tags/blonde/0.jpeg index db7a33cf..a873bfe1 100644 Binary files a/public/img/tags/blonde/0.jpeg and b/public/img/tags/blonde/0.jpeg differ diff --git a/public/img/tags/blonde/lazy/0.jpeg b/public/img/tags/blonde/lazy/0.jpeg index 206e2f6a..3eaf905a 100644 Binary files a/public/img/tags/blonde/lazy/0.jpeg and b/public/img/tags/blonde/lazy/0.jpeg differ diff --git a/public/img/tags/blonde/thumbs/0.jpeg b/public/img/tags/blonde/thumbs/0.jpeg index e7c2be23..fd653082 100644 Binary files a/public/img/tags/blonde/thumbs/0.jpeg and b/public/img/tags/blonde/thumbs/0.jpeg differ diff --git a/public/img/tags/fake-boobs/2.jpeg b/public/img/tags/fake-boobs/2.jpeg new file mode 100644 index 00000000..10d3e818 Binary files /dev/null and b/public/img/tags/fake-boobs/2.jpeg differ diff --git a/public/img/tags/fake-boobs/3.jpeg b/public/img/tags/fake-boobs/3.jpeg new file mode 100644 index 00000000..787d2c64 Binary files /dev/null and b/public/img/tags/fake-boobs/3.jpeg differ diff --git a/public/img/tags/fake-boobs/lazy/2.jpeg b/public/img/tags/fake-boobs/lazy/2.jpeg new file mode 100644 index 00000000..931c0c76 Binary files /dev/null and b/public/img/tags/fake-boobs/lazy/2.jpeg differ diff --git a/public/img/tags/fake-boobs/lazy/3.jpeg b/public/img/tags/fake-boobs/lazy/3.jpeg new file mode 100644 index 00000000..9867fc70 Binary files /dev/null and b/public/img/tags/fake-boobs/lazy/3.jpeg differ diff --git a/public/img/tags/fake-boobs/thumbs/0.jpeg b/public/img/tags/fake-boobs/thumbs/0.jpeg index 736e94a1..f0ed227a 100644 Binary files a/public/img/tags/fake-boobs/thumbs/0.jpeg and b/public/img/tags/fake-boobs/thumbs/0.jpeg differ diff --git a/public/img/tags/fake-boobs/thumbs/1.jpeg b/public/img/tags/fake-boobs/thumbs/1.jpeg index 0bbaed1b..634edb36 100644 Binary files a/public/img/tags/fake-boobs/thumbs/1.jpeg and b/public/img/tags/fake-boobs/thumbs/1.jpeg differ diff --git a/public/img/tags/fake-boobs/thumbs/2.jpeg b/public/img/tags/fake-boobs/thumbs/2.jpeg new file mode 100644 index 00000000..87ecb189 Binary files /dev/null and b/public/img/tags/fake-boobs/thumbs/2.jpeg differ diff --git a/public/img/tags/fake-boobs/thumbs/3.jpeg b/public/img/tags/fake-boobs/thumbs/3.jpeg new file mode 100644 index 00000000..de2a9720 Binary files /dev/null and b/public/img/tags/fake-boobs/thumbs/3.jpeg differ diff --git a/public/img/tags/indian/0.jpeg b/public/img/tags/indian/0.jpeg new file mode 100644 index 00000000..419394a9 Binary files /dev/null and b/public/img/tags/indian/0.jpeg differ diff --git a/public/img/tags/indian/lazy/0.jpeg b/public/img/tags/indian/lazy/0.jpeg new file mode 100644 index 00000000..2f143bbb Binary files /dev/null and b/public/img/tags/indian/lazy/0.jpeg differ diff --git a/public/img/tags/indian/thumbs/0.jpeg b/public/img/tags/indian/thumbs/0.jpeg new file mode 100644 index 00000000..1b95783f Binary files /dev/null and b/public/img/tags/indian/thumbs/0.jpeg differ diff --git a/seeds/00_tags.js b/seeds/00_tags.js index c45bf2fc..149affdd 100644 --- a/seeds/00_tags.js +++ b/seeds/00_tags.js @@ -514,6 +514,10 @@ const tags = [ name: 'humiliation', slug: 'humiliation', }, + { + name: 'indian', + slug: 'indian', + }, { name: 'innie pussy', slug: 'innie-pussy', diff --git a/seeds/04_media.js b/seeds/04_media.js index 6a2c31b9..5ab639c8 100644 --- a/seeds/04_media.js +++ b/seeds/04_media.js @@ -594,7 +594,7 @@ const tagPosters = [ ['atogm', 0, 'Alysa Gap and Logan in "Anal Buffet 4" for Evil Angel'], ['bdsm', 0, 'Dani Daniels in "The Traning of Dani Daniels, Day 2" for The Training of O at Kink'], ['behind-the-scenes', 0, 'Janice Griffith in "Day With A Pornstar: Janice" for Brazzers'], - ['blonde', 0, 'Anikka Albrite for Digital Playground'], + ['blonde', 0, 'Anikka Albrite in "Black Owned 4" for Jules Jordan'], ['blowbang', 0, 'Lacy Lennon in "Lacy Lennon\'s First Blowbang" for HardX'], ['blowjob', 0, 'Adriana Chechik in "The Dinner Party" for Real Wife Stories (Brazzers)'], ['brunette', 0, 'Nicole Black in GIO971 for LegalPorno'], @@ -615,11 +615,12 @@ const tagPosters = [ ['ebony', 1, 'Ana Foxxx in "DP Me 4" for HardX'], ['facefucking', 2, 'Jynx Maze for Throated'], ['facial', 0, 'Brooklyn Gray in "All About Ass 4" for Evil Angel'], - ['fake-boobs', 1, 'Lela Star in "Thick" for Jules Jordan'], + ['fake-boobs', 2, 'Gia Milana in "Hot Anal Latina" for HardX'], ['family', 0, 'Teanna Trump in "A Family Appear: Part One" for Brazzers'], ['femdom', 0, 'Alina Li in "Asian Domination… She Holds Jules Jordan\'s Cock Hostage!" for Jules Jordan'], ['gangbang', 5, 'Carter Cruise\'s first gangbang in "Slut Puppies 9" for Jules Jordan'], ['gaping', 1, 'Vina Sky in "Vina Sky Does Anal" for HardX'], + ['indian', 0, 'Resha in "Casting Resha" for Watch 4 Beauty'], ['interracial', 0, 'Jaye Summers and Prince Yahshua in "Platinum Pussy 3" for Jules Jordan'], ['latina', 1, 'Jynx Maze in "Big Anal Asses 2" for HardX'], ['lesbian', 0, 'Jenna Sativa and Alina Lopez in "Opposites Attract" for Girl Girl'], @@ -699,6 +700,8 @@ const tagPhotos = [ ['facial', 'poster', 'Jynx Maze'], ['facefucking', 3, 'Adriana Chechik in "Performing Magic Butt Tricks With Jules Jordan. What Will Disappear In Her Ass?" for Jules Jordan'], ['facefucking', 1, 'Carrie for Young Throats'], + ['fake-boobs', 1, 'Lela Star in "Thick" for Jules Jordan'], + ['fake-boobs', 3, 'Ashly Anderson for Passion HD'], // ['fake-boobs', 0, 'Marsha May in "Once You Go Black 7" for Jules Jordan'], ['gangbang', 'poster', 'Kristen Scott in "Interracial Gangbang!" for Jules Jordan'], ['gangbang', 0, '"4 On 1 Gangbangs" for Doghouse Digital'], diff --git a/src/actors.js b/src/actors.js index 5f4f0ae4..a7706b7a 100644 --- a/src/actors.js +++ b/src/actors.js @@ -134,7 +134,7 @@ function toBaseActors(actorsOrNames, release) { }); } -function curateActor(actor, withDetails = false) { +function curateActor(actor, withDetails = false, isProfile = false) { if (!actor) { return null; } @@ -174,7 +174,7 @@ function curateActor(actor, withDetails = false) { hasPiercings: actor.has_piercings, tattoos: actor.tattoos, piercings: actor.piercings, - description: actor.description, + ...(isProfile && { description: actor.description }), placeOfBirth: actor.birth_country && { country: { alpha2: actor.birth_country.alpha2, @@ -201,6 +201,7 @@ function curateActor(actor, withDetails = false) { size: actor.avatar.size, source: actor.avatar.source, }, + ...(actor.profiles && { profiles: actor.profiles?.map(profile => curateActor(profile, true, true)) }), }), }; @@ -504,12 +505,14 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy return await [].concat(source).reduce(async (outcome, scraperSlug) => outcome.catch(async () => { try { const scraper = scrapers[scraperSlug]; + const entity = entitiesBySlug[scraperSlug] || null; + const context = { - ...entitiesBySlug[scraperSlug], + ...entity, // legacy - site: entitiesBySlug[scraperSlug] || null, - network: entitiesBySlug[scraperSlug] || null, - entity: entitiesBySlug[scraperSlug] || null, + site: entity, + network: entity?.parent, + entity, scraper: scraperSlug, }; @@ -547,7 +550,7 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy return await curateProfile({ ...actor, ...profile, - ...context, + entity, update: existingProfile?.id || false, }); } catch (error) { @@ -715,7 +718,8 @@ async function fetchActor(actorId) { row_to_json(actor_alias) as alias, row_to_json(birth_country) as birth_country, row_to_json(residence_country) as residence_country, - row_to_json(media) as avatar + row_to_json(media) as avatar, + json_agg(actors_profiles) as profiles `)) .modify((queryBuilder) => { if (Number.isNaN(Number(actorId))) { @@ -726,10 +730,12 @@ async function fetchActor(actorId) { queryBuilder.where('actors.id', actorId); }) .leftJoin('actors as actor_alias', 'actor_alias.id', 'actors.alias_for') + .leftJoin('actors_profiles', 'actors.id', 'actors_profiles.actor_id') .leftJoin('entities', 'entities.id', 'actors.entity_id') .leftJoin('countries as birth_country', 'birth_country.alpha2', 'actors.birth_country_alpha2') .leftJoin('countries as residence_country', 'residence_country.alpha2', 'actors.residence_country_alpha2') .leftJoin('media', 'media.id', 'actors.avatar_media_id') + .groupBy('actors.id', 'entities.id', 'actor_alias.id', 'birth_country.alpha2', 'residence_country.alpha2', 'media.id') .first(); return curateActor(actor, true); diff --git a/src/media.js b/src/media.js index ac42457d..766e75de 100644 --- a/src/media.js +++ b/src/media.js @@ -10,7 +10,7 @@ const stream = require('stream'); const nanoid = require('nanoid/non-secure'); const mime = require('mime'); // const fileType = require('file-type'); -const youtubeDl = require('youtube-dl'); +const ffmpeg = require('fluent-ffmpeg'); const sharp = require('sharp'); const blake2 = require('blake2'); @@ -418,22 +418,21 @@ async function fetchHttpSource(source, tempFileTarget, hashStream) { }; } -async function fetchStreamSource(source, tempFileTarget, hashStream) { - const video = youtubeDl(source.stream); +async function fetchStreamSource(source, tempFileTarget, tempFilePath, hashStream) { + const meta = { mimetype: 'video/mp4' }; - video.on('info', (info) => { - console.log(info); - logger.verbose(`Starting fetching stream from ${source.stream}`); - }); - - video.on('end', (info) => { - console.log(info); - logger.verbose(`Finished fetching stream from ${source.stream}`); - }); + const video = ffmpeg(source.stream) + .format('mp4') + .outputOptions(['-movflags frag_keyframe+empty_moov']) + .on('start', cmd => logger.verbose(`Fetching stream from ${source.stream} with "${cmd}"`)) + .on('error', error => logger.error(`Failed to fetch stream from ${source.stream}: ${error.message}`)) + .pipe(); await pipeline(video, hashStream, tempFileTarget); - return { mimetype: null }; + logger.verbose(`Finished fetching stream from ${source.stream}`); + + return meta; } async function fetchSource(source, baseMedia) { @@ -457,7 +456,7 @@ async function fetchSource(source, baseMedia) { }); const { mimetype } = source.stream - ? await fetchStreamSource(source, tempFileTarget, hashStream) + ? await fetchStreamSource(source, tempFileTarget, tempFilePath, hashStream) : await fetchHttpSource(source, tempFileTarget, hashStream); hasher.end(); diff --git a/src/releases.js b/src/releases.js index 922878b7..36f1803b 100644 --- a/src/releases.js +++ b/src/releases.js @@ -7,8 +7,6 @@ function curateRelease(release, withMedia = false) { return null; } - const network = release.site_network || release.network; - return { id: release.id, entryId: release.entry_id, @@ -18,22 +16,22 @@ function curateRelease(release, withMedia = false) { date: release.date, description: release.description, duration: release.duration, - site: release.site && { - id: release.site.id, - name: release.site.name, - slug: release.site.slug, - }, - network: network && { - id: network.id, - name: network.name, - slug: network.slug, + entity: release.entity && { + id: release.entity.id, + name: release.entity.name, + slug: release.entity.slug, + parent: release.entity.parent && { + id: release.entity.parent.id, + name: release.entity.parent.name, + slug: release.entity.parent.slug, + }, }, actors: (release.actors || []).map(actor => ({ id: actor.id, name: actor.name, slug: actor.slug, gender: actor.gender, - networkId: actor.network_id, + entityId: actor.entity_id, aliasFor: actor.alias_for, })), tags: (release.tags || []).map(tag => ({ @@ -67,23 +65,21 @@ function withRelations(queryBuilder, withMedia = false, type = 'scene') { queryBuilder .select(knex.raw(` releases.id, releases.entry_id, releases.shoot_id, releases.title, releases.url, releases.date, releases.description, releases.duration, releases.created_at, - row_to_json(sites) as site, - row_to_json(networks) as network, - row_to_json(site_networks) as site_network, + row_to_json(entities) as entity, + row_to_json(parents) as parent, COALESCE(json_agg(DISTINCT actors) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors, COALESCE(json_agg(DISTINCT tags) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags `)) - .where('type', type) - .leftJoin('sites', 'sites.id', 'releases.site_id') - .leftJoin('networks', 'networks.id', 'releases.network_id') - .leftJoin('networks as site_networks', 'site_networks.id', 'sites.network_id') + .where('releases.type', type) + .leftJoin('entities', 'entities.id', 'releases.entity_id') + .leftJoin('entities as parents', 'parents.id', 'entities.parent_id') .leftJoin('releases_actors', 'releases_actors.release_id', 'releases.id') .leftJoin('actors', 'actors.id', 'releases_actors.actor_id') .leftJoin('releases_tags', 'releases_tags.release_id', 'releases.id') .leftJoin('tags', 'tags.id', 'releases_tags.tag_id') .groupBy(knex.raw(` releases.id, releases.entry_id, releases.shoot_id, releases.title, releases.url, releases.date, releases.description, releases.duration, releases.created_at, - sites.id, networks.id, site_networks.id + entities.id, parents.id `)); if (withMedia) { diff --git a/src/scrapers/elegantangel.js b/src/scrapers/elegantangel.js new file mode 100644 index 00000000..c2f349c6 --- /dev/null +++ b/src/scrapers/elegantangel.js @@ -0,0 +1,136 @@ +'use strict'; + +const qu = require('../utils/q'); +const slugify = require('../utils/slugify'); + +function scrapeAll(scenes, channel) { + return scenes.map(({ query }) => { + const release = {}; + + release.url = query.url('.scene-update-details, .feature-update-details', 'href', { origin: channel.url }); + release.entryId = new URL(release.url).pathname.match(/\/(\d+)/)[1]; + + release.title = query.q('.scene-img-wrapper img', 'alt').replace(/\s*image$/i, ''); + + release.date = query.date('.scene-update-stats span, .feature-update-details span', 'MMM DD, YYYY'); + release.actors = query.cnt('.scene-update-details h3, .feature-update-details h2')?.split(/\s*\|\s*/).map(actor => actor.trim()); + + const poster = query.img('.scene-img-wrapper img'); + release.poster = [ + poster.replace(/\/res\/\d+/, '/res/1920'), + poster.replace(/\/res\/\d+/, '/res/1600'), + poster, + ]; + + release.trailer = { src: query.video('.scene-img-wrapper source') }; + + return release; + }); +} + +async function scrapeScene({ query, html }, url) { + const release = {}; + + release.entryId = new URL(url).pathname.match(/\/(\d+)/)[1]; + + release.title = query.cnt('.scene-page .description'); + release.date = query.date('.release-date:first-child', 'MMM DD, YYYY', /\w{3} \d{2}, \d{4}/); + release.duration = query.number('.release-date:last-child') * 60; + + release.actors = query.all('.video-performer').map((el) => { + const avatar = qu.query.img(el, 'img', 'data-bgsrc'); + + return { + name: qu.query.cnt(el, 'span'), + avatar: [ + avatar.replace(/\/actor\/\d+/, '/actor/1600'), + avatar, + ], + }; + }); + + release.tags = query.cnts('.tags a'); + release.poster = query.url('link[rel="image_src"]') || query.meta('property="og:image"'); + + release.photos = query.imgs('#dv_frames a > img').map(photo => [ + photo.replace(/(\/p\/\d+\/)\d+/, (match, path) => `${path}1920`), + photo.replace(/(\/p\/\d+\/)\d+/, (match, path) => `${path}1600`), + photo, + ]); + + const trailerId = html.match(/item: (\d+),/)?.[1]; + + if (trailerId) { + const trailerUrl = `https://www.adultempire.com/videoEmbed/${trailerId}?type=preview`; + const trailerRes = await qu.get(trailerUrl); + + if (trailerRes.ok) { + const stream = trailerRes.item.query.video(); + + release.trailer = { stream }; + } + } + + // console.log(release); + return release; +} + +function scrapeProfile({ query, el }, actorName, entity, include) { + const profile = {}; + + profile.description = query.cnt('.bio-text'); + profile.birthPlace = query.cnt('.birth-place span'); + + profile.avatar = query.img('.actor-photo img'); + + if (include.releases) { + return scrapeAll(qu.initAll(el, '.scene')); + } + + console.log(profile); + return profile; +} + +async function fetchLatest(channel, page = 1) { + const url = `${channel.url}/tour?page=${page}`; + const res = await qu.getAll(url, '.scene-update', null, { + // invalid certificate + rejectUnauthorized: false, + }); + + if (res.ok) { + return scrapeAll(res.items, channel); + } + + return res.status; +} + +async function fetchScene(url, channel) { + const res = await qu.get(url, null, null, { + // invalid certificate + rejectUnauthorized: false, + }); + + if (res.ok) { + return scrapeScene(res.item, url, channel); + } + + return res.status; +} + +async function fetchProfile(actorName, entity, include) { + const url = `${entity.url}/actors/${slugify(actorName, '_')}`; + const res = await qu.get(url); + + if (res.ok) { + return scrapeProfile(res.item, actorName, entity, include); + } + + return res.status; +} + +module.exports = { + fetchLatest, + fetchScene, + fetchProfile, +}; diff --git a/src/scrapers/template.js b/src/scrapers/template.js index e72475dc..0416e6c4 100644 --- a/src/scrapers/template.js +++ b/src/scrapers/template.js @@ -3,13 +3,12 @@ const qu = require('../utils/q'); const slugify = require('../utils/slugify'); -function scrapeAll(scenes, site) { +function scrapeAll(scenes) { return scenes.map(({ query }) => { const release = {}; - const pathname = query.url('.title a'); - release.entryId = pathname.match(/\/scene\/(\d+)/)[1]; - release.url = `${site.url}${pathname}`; + release.url = query.url('.title a'); + release.entryId = new URL(release.url).pathname.match(/\/scene\/(\d+)/)[1]; release.title = query.cnt('.title a'); release.description = query.cnt('.description'); @@ -17,7 +16,8 @@ function scrapeAll(scenes, site) { release.date = query.date('.date', 'MMM DD, YYYY'); release.actors = query.cnts('.models a.model'); - release.poster = query.q('img.poster'); + release.poster = query.img('img.poster'); + release.teaser = { src: query.video('.teaser video') }; release.stars = query.number('.rating'); release.likes = query.number('.likes'); @@ -27,16 +27,16 @@ function scrapeAll(scenes, site) { }); } -function scrapeScene({ query }) { +function scrapeScene({ query }, url) { const release = {}; + release.entryId = new URL(url).pathname.match(/\/scene\/(\d+)/)[1]; + release.title = query.cnt('h3.title'); release.description = query.cnt('p.description'); [release.poster, ...release.photos] = query.imgs('.preview-thumb'); - - const trailer = query.video('.trailer video'); - release.trailer = { src: trailer }; + release.trailer = { src: query.video('.trailer video') }; console.log(release); return release; @@ -73,7 +73,7 @@ async function fetchScene(url, channel) { const res = await qu.get(url); if (res.ok) { - return scrapeScene(res.item, channel); + return scrapeScene(res.item, url, channel); } return res.status; diff --git a/src/utils/qu.js b/src/utils/qu.js index 5d8058bd..d7ff62dc 100644 --- a/src/utils/qu.js +++ b/src/utils/qu.js @@ -140,10 +140,18 @@ function styles(context, selector, styleAttr) { return elStyles; } -function number(context, selector, attr = true) { +function number(context, selector, match = /\d+/, attr = 'textContent') { const value = q(context, selector, attr); - return value ? Number(value) : null; + if (value && match) { + return Number(value.match(match)?.[0]); + } + + if (value) { + return Number(value); + } + + return null; } function meta(context, selector, attrArg = 'content', applyTrim = true) { @@ -280,6 +288,8 @@ const quFuncs = { date, dur: duration, duration, + element: q, + el: q, exists, image, images,