Merge branch 'experimental'
This commit is contained in:
DebaucheryLibrarian
commit
0b57ebb10f
|
|
@ -52,6 +52,7 @@ async function mounted() {
|
|||
'ebony',
|
||||
'latina',
|
||||
'caucasian',
|
||||
'indian',
|
||||
'natural-boobs',
|
||||
'fake-boobs',
|
||||
'blonde',
|
||||
|
|
|
|||
|
|
@ -4614,6 +4614,15 @@
|
|||
"integrity": "sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg==",
|
||||
"dev": true
|
||||
},
|
||||
"fluent-ffmpeg": {
|
||||
"version": "2.1.2",
|
||||
"resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz",
|
||||
"integrity": "sha1-yVLeIkD4EuvaCqgAbXd27irPfXQ=",
|
||||
"requires": {
|
||||
"async": ">=0.2.9",
|
||||
"which": "^1.1.1"
|
||||
}
|
||||
},
|
||||
"flush-write-stream": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-1.1.1.tgz",
|
||||
|
|
@ -5721,28 +5730,12 @@
|
|||
"minimalistic-assert": "^1.0.1"
|
||||
}
|
||||
},
|
||||
"hashish": {
|
||||
"version": "0.0.4",
|
||||
"resolved": "https://registry.npmjs.org/hashish/-/hashish-0.0.4.tgz",
|
||||
"integrity": "sha1-bWC8b/r3Ebav1g5CbQd5iAFOZVQ=",
|
||||
"requires": {
|
||||
"traverse": ">=0.2.4"
|
||||
}
|
||||
},
|
||||
"he": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
|
||||
"integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==",
|
||||
"dev": true
|
||||
},
|
||||
"hh-mm-ss": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/hh-mm-ss/-/hh-mm-ss-1.2.0.tgz",
|
||||
"integrity": "sha512-f4I9Hz1dLpX/3mrEs7yq30+FiuO3tt5NWAqAGeBTaoeoBfB8vhcQ3BphuDc5DjZb/K809agqrAaFlP0jhEU/8w==",
|
||||
"requires": {
|
||||
"zero-fill": "^2.2.3"
|
||||
}
|
||||
},
|
||||
"hmac-drbg": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz",
|
||||
|
|
@ -5845,11 +5838,6 @@
|
|||
"debug": "^3.1.0"
|
||||
}
|
||||
},
|
||||
"human-signals": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/human-signals/-/human-signals-1.1.1.tgz",
|
||||
"integrity": "sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw=="
|
||||
},
|
||||
"iconv-lite": {
|
||||
"version": "0.5.1",
|
||||
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.1.tgz",
|
||||
|
|
@ -7168,11 +7156,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"merge-stream": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
|
||||
"integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="
|
||||
},
|
||||
"mersenne-twister": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/mersenne-twister/-/mersenne-twister-1.1.0.tgz",
|
||||
|
|
@ -10456,14 +10439,6 @@
|
|||
"mixme": "^0.3.1"
|
||||
}
|
||||
},
|
||||
"streamify": {
|
||||
"version": "0.2.9",
|
||||
"resolved": "https://registry.npmjs.org/streamify/-/streamify-0.2.9.tgz",
|
||||
"integrity": "sha512-8pUxeLEef9UO1FxtTt5iikAiyzGI4SZRnGuJ3sz8axZ5Xk+/7ezEV5kuJQsMEFxw7AKYw3xp0Ow+20mmSaJbQQ==",
|
||||
"requires": {
|
||||
"hashish": "~0.0.4"
|
||||
}
|
||||
},
|
||||
"strict-uri-encode": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz",
|
||||
|
|
@ -10542,11 +10517,6 @@
|
|||
"integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=",
|
||||
"dev": true
|
||||
},
|
||||
"strip-final-newline": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz",
|
||||
"integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA=="
|
||||
},
|
||||
"strip-indent": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-1.0.1.tgz",
|
||||
|
|
@ -10966,11 +10936,6 @@
|
|||
"punycode": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"traverse": {
|
||||
"version": "0.6.6",
|
||||
"resolved": "https://registry.npmjs.org/traverse/-/traverse-0.6.6.tgz",
|
||||
"integrity": "sha1-y99WD9e5r2MlAv7UD5GMFX6pcTc="
|
||||
},
|
||||
"trim-newlines": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz",
|
||||
|
|
@ -12110,127 +12075,6 @@
|
|||
"camelcase": "^5.0.0",
|
||||
"decamelize": "^1.2.0"
|
||||
}
|
||||
},
|
||||
"youtube-dl": {
|
||||
"version": "3.0.2",
|
||||
"resolved": "https://registry.npmjs.org/youtube-dl/-/youtube-dl-3.0.2.tgz",
|
||||
"integrity": "sha512-LFFfpsYbRLpqKsnb4gzbnyN7fm190tJw3gJVSvfoEfnb/xYIPNT6i9G3jdzPDp/U5cwB3OSq63nUa7rUwxXAGA==",
|
||||
"requires": {
|
||||
"debug": "~4.1.1",
|
||||
"execa": "~3.2.0",
|
||||
"hh-mm-ss": "~1.2.0",
|
||||
"mkdirp": "~0.5.1",
|
||||
"request": "~2.88.0",
|
||||
"streamify": "~0.2.9",
|
||||
"universalify": "~0.1.2"
|
||||
},
|
||||
"dependencies": {
|
||||
"cross-spawn": {
|
||||
"version": "7.0.3",
|
||||
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
|
||||
"integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
|
||||
"requires": {
|
||||
"path-key": "^3.1.0",
|
||||
"shebang-command": "^2.0.0",
|
||||
"which": "^2.0.1"
|
||||
}
|
||||
},
|
||||
"debug": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
|
||||
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
|
||||
"requires": {
|
||||
"ms": "^2.1.1"
|
||||
}
|
||||
},
|
||||
"execa": {
|
||||
"version": "3.2.0",
|
||||
"resolved": "https://registry.npmjs.org/execa/-/execa-3.2.0.tgz",
|
||||
"integrity": "sha512-kJJfVbI/lZE1PZYDI5VPxp8zXPO9rtxOkhpZ0jMKha56AI9y2gGVC6bkukStQf0ka5Rh15BA5m7cCCH4jmHqkw==",
|
||||
"requires": {
|
||||
"cross-spawn": "^7.0.0",
|
||||
"get-stream": "^5.0.0",
|
||||
"human-signals": "^1.1.1",
|
||||
"is-stream": "^2.0.0",
|
||||
"merge-stream": "^2.0.0",
|
||||
"npm-run-path": "^4.0.0",
|
||||
"onetime": "^5.1.0",
|
||||
"p-finally": "^2.0.0",
|
||||
"signal-exit": "^3.0.2",
|
||||
"strip-final-newline": "^2.0.0"
|
||||
}
|
||||
},
|
||||
"get-stream": {
|
||||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz",
|
||||
"integrity": "sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==",
|
||||
"requires": {
|
||||
"pump": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"is-stream": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz",
|
||||
"integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw=="
|
||||
},
|
||||
"mimic-fn": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz",
|
||||
"integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg=="
|
||||
},
|
||||
"npm-run-path": {
|
||||
"version": "4.0.1",
|
||||
"resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz",
|
||||
"integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==",
|
||||
"requires": {
|
||||
"path-key": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"onetime": {
|
||||
"version": "5.1.0",
|
||||
"resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.0.tgz",
|
||||
"integrity": "sha512-5NcSkPHhwTVFIQN+TUqXoS5+dlElHXdpAWu9I0HP20YOtIi+aZ0Ct82jdlILDxjLEAWwvm+qj1m6aEtsDVmm6Q==",
|
||||
"requires": {
|
||||
"mimic-fn": "^2.1.0"
|
||||
}
|
||||
},
|
||||
"p-finally": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/p-finally/-/p-finally-2.0.1.tgz",
|
||||
"integrity": "sha512-vpm09aKwq6H9phqRQzecoDpD8TmVyGw70qmWlyq5onxY7tqyTTFVvxMykxQSQKILBSFlbXpypIw2T1Ml7+DDtw=="
|
||||
},
|
||||
"path-key": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
|
||||
"integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="
|
||||
},
|
||||
"shebang-command": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
|
||||
"integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
|
||||
"requires": {
|
||||
"shebang-regex": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"shebang-regex": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
|
||||
"integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="
|
||||
},
|
||||
"which": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
|
||||
"integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
|
||||
"requires": {
|
||||
"isexe": "^2.0.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"zero-fill": {
|
||||
"version": "2.2.3",
|
||||
"resolved": "https://registry.npmjs.org/zero-fill/-/zero-fill-2.2.3.tgz",
|
||||
"integrity": "sha1-o97wa6XjmuZEhQu0yirUEStIVek="
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -91,6 +91,7 @@
|
|||
"express-react-views": "^0.11.0",
|
||||
"face-api.js": "^0.21.0",
|
||||
"file-type": "^14.1.4",
|
||||
"fluent-ffmpeg": "^2.1.2",
|
||||
"fs-extra": "^7.0.1",
|
||||
"graphile-utils": "^4.5.6",
|
||||
"iconv-lite": "^0.5.1",
|
||||
|
|
@ -124,7 +125,6 @@
|
|||
"vuex": "^3.1.2",
|
||||
"winston": "^3.2.1",
|
||||
"winston-daily-rotate-file": "^4.4.2",
|
||||
"yargs": "^13.3.0",
|
||||
"youtube-dl": "^3.0.2"
|
||||
"yargs": "^13.3.0"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Binary file not shown.
|
After Width: | Height: | Size: 1.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.7 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 253 KiB After Width: | Height: | Size: 792 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 7.3 KiB After Width: | Height: | Size: 9.1 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 40 KiB After Width: | Height: | Size: 44 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.3 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 356 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 7.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 7.1 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 46 KiB After Width: | Height: | Size: 36 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 47 KiB After Width: | Height: | Size: 37 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 29 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 28 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.6 MiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
|
|
@ -514,6 +514,10 @@ const tags = [
|
|||
name: 'humiliation',
|
||||
slug: 'humiliation',
|
||||
},
|
||||
{
|
||||
name: 'indian',
|
||||
slug: 'indian',
|
||||
},
|
||||
{
|
||||
name: 'innie pussy',
|
||||
slug: 'innie-pussy',
|
||||
|
|
|
|||
|
|
@ -594,7 +594,7 @@ const tagPosters = [
|
|||
['atogm', 0, 'Alysa Gap and Logan in "Anal Buffet 4" for Evil Angel'],
|
||||
['bdsm', 0, 'Dani Daniels in "The Traning of Dani Daniels, Day 2" for The Training of O at Kink'],
|
||||
['behind-the-scenes', 0, 'Janice Griffith in "Day With A Pornstar: Janice" for Brazzers'],
|
||||
['blonde', 0, 'Anikka Albrite for Digital Playground'],
|
||||
['blonde', 0, 'Anikka Albrite in "Black Owned 4" for Jules Jordan'],
|
||||
['blowbang', 0, 'Lacy Lennon in "Lacy Lennon\'s First Blowbang" for HardX'],
|
||||
['blowjob', 0, 'Adriana Chechik in "The Dinner Party" for Real Wife Stories (Brazzers)'],
|
||||
['brunette', 0, 'Nicole Black in GIO971 for LegalPorno'],
|
||||
|
|
@ -615,11 +615,12 @@ const tagPosters = [
|
|||
['ebony', 1, 'Ana Foxxx in "DP Me 4" for HardX'],
|
||||
['facefucking', 2, 'Jynx Maze for Throated'],
|
||||
['facial', 0, 'Brooklyn Gray in "All About Ass 4" for Evil Angel'],
|
||||
['fake-boobs', 1, 'Lela Star in "Thick" for Jules Jordan'],
|
||||
['fake-boobs', 2, 'Gia Milana in "Hot Anal Latina" for HardX'],
|
||||
['family', 0, 'Teanna Trump in "A Family Appear: Part One" for Brazzers'],
|
||||
['femdom', 0, 'Alina Li in "Asian Domination… She Holds Jules Jordan\'s Cock Hostage!" for Jules Jordan'],
|
||||
['gangbang', 5, 'Carter Cruise\'s first gangbang in "Slut Puppies 9" for Jules Jordan'],
|
||||
['gaping', 1, 'Vina Sky in "Vina Sky Does Anal" for HardX'],
|
||||
['indian', 0, 'Resha in "Casting Resha" for Watch 4 Beauty'],
|
||||
['interracial', 0, 'Jaye Summers and Prince Yahshua in "Platinum Pussy 3" for Jules Jordan'],
|
||||
['latina', 1, 'Jynx Maze in "Big Anal Asses 2" for HardX'],
|
||||
['lesbian', 0, 'Jenna Sativa and Alina Lopez in "Opposites Attract" for Girl Girl'],
|
||||
|
|
@ -699,6 +700,8 @@ const tagPhotos = [
|
|||
['facial', 'poster', 'Jynx Maze'],
|
||||
['facefucking', 3, 'Adriana Chechik in "Performing Magic Butt Tricks With Jules Jordan. What Will Disappear In Her Ass?" for Jules Jordan'],
|
||||
['facefucking', 1, 'Carrie for Young Throats'],
|
||||
['fake-boobs', 1, 'Lela Star in "Thick" for Jules Jordan'],
|
||||
['fake-boobs', 3, 'Ashly Anderson for Passion HD'],
|
||||
// ['fake-boobs', 0, 'Marsha May in "Once You Go Black 7" for Jules Jordan'],
|
||||
['gangbang', 'poster', 'Kristen Scott in "Interracial Gangbang!" for Jules Jordan'],
|
||||
['gangbang', 0, '"4 On 1 Gangbangs" for Doghouse Digital'],
|
||||
|
|
|
|||
|
|
@ -134,7 +134,7 @@ function toBaseActors(actorsOrNames, release) {
|
|||
});
|
||||
}
|
||||
|
||||
function curateActor(actor, withDetails = false) {
|
||||
function curateActor(actor, withDetails = false, isProfile = false) {
|
||||
if (!actor) {
|
||||
return null;
|
||||
}
|
||||
|
|
@ -174,7 +174,7 @@ function curateActor(actor, withDetails = false) {
|
|||
hasPiercings: actor.has_piercings,
|
||||
tattoos: actor.tattoos,
|
||||
piercings: actor.piercings,
|
||||
description: actor.description,
|
||||
...(isProfile && { description: actor.description }),
|
||||
placeOfBirth: actor.birth_country && {
|
||||
country: {
|
||||
alpha2: actor.birth_country.alpha2,
|
||||
|
|
@ -201,6 +201,7 @@ function curateActor(actor, withDetails = false) {
|
|||
size: actor.avatar.size,
|
||||
source: actor.avatar.source,
|
||||
},
|
||||
...(actor.profiles && { profiles: actor.profiles?.map(profile => curateActor(profile, true, true)) }),
|
||||
}),
|
||||
};
|
||||
|
||||
|
|
@ -504,12 +505,14 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
|
|||
return await [].concat(source).reduce(async (outcome, scraperSlug) => outcome.catch(async () => {
|
||||
try {
|
||||
const scraper = scrapers[scraperSlug];
|
||||
const entity = entitiesBySlug[scraperSlug] || null;
|
||||
|
||||
const context = {
|
||||
...entitiesBySlug[scraperSlug],
|
||||
...entity,
|
||||
// legacy
|
||||
site: entitiesBySlug[scraperSlug] || null,
|
||||
network: entitiesBySlug[scraperSlug] || null,
|
||||
entity: entitiesBySlug[scraperSlug] || null,
|
||||
site: entity,
|
||||
network: entity?.parent,
|
||||
entity,
|
||||
scraper: scraperSlug,
|
||||
};
|
||||
|
||||
|
|
@ -547,7 +550,7 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
|
|||
return await curateProfile({
|
||||
...actor,
|
||||
...profile,
|
||||
...context,
|
||||
entity,
|
||||
update: existingProfile?.id || false,
|
||||
});
|
||||
} catch (error) {
|
||||
|
|
@ -715,7 +718,8 @@ async function fetchActor(actorId) {
|
|||
row_to_json(actor_alias) as alias,
|
||||
row_to_json(birth_country) as birth_country,
|
||||
row_to_json(residence_country) as residence_country,
|
||||
row_to_json(media) as avatar
|
||||
row_to_json(media) as avatar,
|
||||
json_agg(actors_profiles) as profiles
|
||||
`))
|
||||
.modify((queryBuilder) => {
|
||||
if (Number.isNaN(Number(actorId))) {
|
||||
|
|
@ -726,10 +730,12 @@ async function fetchActor(actorId) {
|
|||
queryBuilder.where('actors.id', actorId);
|
||||
})
|
||||
.leftJoin('actors as actor_alias', 'actor_alias.id', 'actors.alias_for')
|
||||
.leftJoin('actors_profiles', 'actors.id', 'actors_profiles.actor_id')
|
||||
.leftJoin('entities', 'entities.id', 'actors.entity_id')
|
||||
.leftJoin('countries as birth_country', 'birth_country.alpha2', 'actors.birth_country_alpha2')
|
||||
.leftJoin('countries as residence_country', 'residence_country.alpha2', 'actors.residence_country_alpha2')
|
||||
.leftJoin('media', 'media.id', 'actors.avatar_media_id')
|
||||
.groupBy('actors.id', 'entities.id', 'actor_alias.id', 'birth_country.alpha2', 'residence_country.alpha2', 'media.id')
|
||||
.first();
|
||||
|
||||
return curateActor(actor, true);
|
||||
|
|
|
|||
27
src/media.js
27
src/media.js
|
|
@ -10,7 +10,7 @@ const stream = require('stream');
|
|||
const nanoid = require('nanoid/non-secure');
|
||||
const mime = require('mime');
|
||||
// const fileType = require('file-type');
|
||||
const youtubeDl = require('youtube-dl');
|
||||
const ffmpeg = require('fluent-ffmpeg');
|
||||
const sharp = require('sharp');
|
||||
const blake2 = require('blake2');
|
||||
|
||||
|
|
@ -418,22 +418,21 @@ async function fetchHttpSource(source, tempFileTarget, hashStream) {
|
|||
};
|
||||
}
|
||||
|
||||
async function fetchStreamSource(source, tempFileTarget, hashStream) {
|
||||
const video = youtubeDl(source.stream);
|
||||
async function fetchStreamSource(source, tempFileTarget, tempFilePath, hashStream) {
|
||||
const meta = { mimetype: 'video/mp4' };
|
||||
|
||||
video.on('info', (info) => {
|
||||
console.log(info);
|
||||
logger.verbose(`Starting fetching stream from ${source.stream}`);
|
||||
});
|
||||
|
||||
video.on('end', (info) => {
|
||||
console.log(info);
|
||||
logger.verbose(`Finished fetching stream from ${source.stream}`);
|
||||
});
|
||||
const video = ffmpeg(source.stream)
|
||||
.format('mp4')
|
||||
.outputOptions(['-movflags frag_keyframe+empty_moov'])
|
||||
.on('start', cmd => logger.verbose(`Fetching stream from ${source.stream} with "${cmd}"`))
|
||||
.on('error', error => logger.error(`Failed to fetch stream from ${source.stream}: ${error.message}`))
|
||||
.pipe();
|
||||
|
||||
await pipeline(video, hashStream, tempFileTarget);
|
||||
|
||||
return { mimetype: null };
|
||||
logger.verbose(`Finished fetching stream from ${source.stream}`);
|
||||
|
||||
return meta;
|
||||
}
|
||||
|
||||
async function fetchSource(source, baseMedia) {
|
||||
|
|
@ -457,7 +456,7 @@ async function fetchSource(source, baseMedia) {
|
|||
});
|
||||
|
||||
const { mimetype } = source.stream
|
||||
? await fetchStreamSource(source, tempFileTarget, hashStream)
|
||||
? await fetchStreamSource(source, tempFileTarget, tempFilePath, hashStream)
|
||||
: await fetchHttpSource(source, tempFileTarget, hashStream);
|
||||
|
||||
hasher.end();
|
||||
|
|
|
|||
|
|
@ -7,8 +7,6 @@ function curateRelease(release, withMedia = false) {
|
|||
return null;
|
||||
}
|
||||
|
||||
const network = release.site_network || release.network;
|
||||
|
||||
return {
|
||||
id: release.id,
|
||||
entryId: release.entry_id,
|
||||
|
|
@ -18,22 +16,22 @@ function curateRelease(release, withMedia = false) {
|
|||
date: release.date,
|
||||
description: release.description,
|
||||
duration: release.duration,
|
||||
site: release.site && {
|
||||
id: release.site.id,
|
||||
name: release.site.name,
|
||||
slug: release.site.slug,
|
||||
entity: release.entity && {
|
||||
id: release.entity.id,
|
||||
name: release.entity.name,
|
||||
slug: release.entity.slug,
|
||||
parent: release.entity.parent && {
|
||||
id: release.entity.parent.id,
|
||||
name: release.entity.parent.name,
|
||||
slug: release.entity.parent.slug,
|
||||
},
|
||||
network: network && {
|
||||
id: network.id,
|
||||
name: network.name,
|
||||
slug: network.slug,
|
||||
},
|
||||
actors: (release.actors || []).map(actor => ({
|
||||
id: actor.id,
|
||||
name: actor.name,
|
||||
slug: actor.slug,
|
||||
gender: actor.gender,
|
||||
networkId: actor.network_id,
|
||||
entityId: actor.entity_id,
|
||||
aliasFor: actor.alias_for,
|
||||
})),
|
||||
tags: (release.tags || []).map(tag => ({
|
||||
|
|
@ -67,23 +65,21 @@ function withRelations(queryBuilder, withMedia = false, type = 'scene') {
|
|||
queryBuilder
|
||||
.select(knex.raw(`
|
||||
releases.id, releases.entry_id, releases.shoot_id, releases.title, releases.url, releases.date, releases.description, releases.duration, releases.created_at,
|
||||
row_to_json(sites) as site,
|
||||
row_to_json(networks) as network,
|
||||
row_to_json(site_networks) as site_network,
|
||||
row_to_json(entities) as entity,
|
||||
row_to_json(parents) as parent,
|
||||
COALESCE(json_agg(DISTINCT actors) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
|
||||
COALESCE(json_agg(DISTINCT tags) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags
|
||||
`))
|
||||
.where('type', type)
|
||||
.leftJoin('sites', 'sites.id', 'releases.site_id')
|
||||
.leftJoin('networks', 'networks.id', 'releases.network_id')
|
||||
.leftJoin('networks as site_networks', 'site_networks.id', 'sites.network_id')
|
||||
.where('releases.type', type)
|
||||
.leftJoin('entities', 'entities.id', 'releases.entity_id')
|
||||
.leftJoin('entities as parents', 'parents.id', 'entities.parent_id')
|
||||
.leftJoin('releases_actors', 'releases_actors.release_id', 'releases.id')
|
||||
.leftJoin('actors', 'actors.id', 'releases_actors.actor_id')
|
||||
.leftJoin('releases_tags', 'releases_tags.release_id', 'releases.id')
|
||||
.leftJoin('tags', 'tags.id', 'releases_tags.tag_id')
|
||||
.groupBy(knex.raw(`
|
||||
releases.id, releases.entry_id, releases.shoot_id, releases.title, releases.url, releases.date, releases.description, releases.duration, releases.created_at,
|
||||
sites.id, networks.id, site_networks.id
|
||||
entities.id, parents.id
|
||||
`));
|
||||
|
||||
if (withMedia) {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,136 @@
|
|||
'use strict';
|
||||
|
||||
const qu = require('../utils/q');
|
||||
const slugify = require('../utils/slugify');
|
||||
|
||||
function scrapeAll(scenes, channel) {
|
||||
return scenes.map(({ query }) => {
|
||||
const release = {};
|
||||
|
||||
release.url = query.url('.scene-update-details, .feature-update-details', 'href', { origin: channel.url });
|
||||
release.entryId = new URL(release.url).pathname.match(/\/(\d+)/)[1];
|
||||
|
||||
release.title = query.q('.scene-img-wrapper img', 'alt').replace(/\s*image$/i, '');
|
||||
|
||||
release.date = query.date('.scene-update-stats span, .feature-update-details span', 'MMM DD, YYYY');
|
||||
release.actors = query.cnt('.scene-update-details h3, .feature-update-details h2')?.split(/\s*\|\s*/).map(actor => actor.trim());
|
||||
|
||||
const poster = query.img('.scene-img-wrapper img');
|
||||
release.poster = [
|
||||
poster.replace(/\/res\/\d+/, '/res/1920'),
|
||||
poster.replace(/\/res\/\d+/, '/res/1600'),
|
||||
poster,
|
||||
];
|
||||
|
||||
release.trailer = { src: query.video('.scene-img-wrapper source') };
|
||||
|
||||
return release;
|
||||
});
|
||||
}
|
||||
|
||||
async function scrapeScene({ query, html }, url) {
|
||||
const release = {};
|
||||
|
||||
release.entryId = new URL(url).pathname.match(/\/(\d+)/)[1];
|
||||
|
||||
release.title = query.cnt('.scene-page .description');
|
||||
release.date = query.date('.release-date:first-child', 'MMM DD, YYYY', /\w{3} \d{2}, \d{4}/);
|
||||
release.duration = query.number('.release-date:last-child') * 60;
|
||||
|
||||
release.actors = query.all('.video-performer').map((el) => {
|
||||
const avatar = qu.query.img(el, 'img', 'data-bgsrc');
|
||||
|
||||
return {
|
||||
name: qu.query.cnt(el, 'span'),
|
||||
avatar: [
|
||||
avatar.replace(/\/actor\/\d+/, '/actor/1600'),
|
||||
avatar,
|
||||
],
|
||||
};
|
||||
});
|
||||
|
||||
release.tags = query.cnts('.tags a');
|
||||
release.poster = query.url('link[rel="image_src"]') || query.meta('property="og:image"');
|
||||
|
||||
release.photos = query.imgs('#dv_frames a > img').map(photo => [
|
||||
photo.replace(/(\/p\/\d+\/)\d+/, (match, path) => `${path}1920`),
|
||||
photo.replace(/(\/p\/\d+\/)\d+/, (match, path) => `${path}1600`),
|
||||
photo,
|
||||
]);
|
||||
|
||||
const trailerId = html.match(/item: (\d+),/)?.[1];
|
||||
|
||||
if (trailerId) {
|
||||
const trailerUrl = `https://www.adultempire.com/videoEmbed/${trailerId}?type=preview`;
|
||||
const trailerRes = await qu.get(trailerUrl);
|
||||
|
||||
if (trailerRes.ok) {
|
||||
const stream = trailerRes.item.query.video();
|
||||
|
||||
release.trailer = { stream };
|
||||
}
|
||||
}
|
||||
|
||||
// console.log(release);
|
||||
return release;
|
||||
}
|
||||
|
||||
function scrapeProfile({ query, el }, actorName, entity, include) {
|
||||
const profile = {};
|
||||
|
||||
profile.description = query.cnt('.bio-text');
|
||||
profile.birthPlace = query.cnt('.birth-place span');
|
||||
|
||||
profile.avatar = query.img('.actor-photo img');
|
||||
|
||||
if (include.releases) {
|
||||
return scrapeAll(qu.initAll(el, '.scene'));
|
||||
}
|
||||
|
||||
console.log(profile);
|
||||
return profile;
|
||||
}
|
||||
|
||||
async function fetchLatest(channel, page = 1) {
|
||||
const url = `${channel.url}/tour?page=${page}`;
|
||||
const res = await qu.getAll(url, '.scene-update', null, {
|
||||
// invalid certificate
|
||||
rejectUnauthorized: false,
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeAll(res.items, channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchScene(url, channel) {
|
||||
const res = await qu.get(url, null, null, {
|
||||
// invalid certificate
|
||||
rejectUnauthorized: false,
|
||||
});
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeScene(res.item, url, channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
async function fetchProfile(actorName, entity, include) {
|
||||
const url = `${entity.url}/actors/${slugify(actorName, '_')}`;
|
||||
const res = await qu.get(url);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeProfile(res.item, actorName, entity, include);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
fetchLatest,
|
||||
fetchScene,
|
||||
fetchProfile,
|
||||
};
|
||||
|
|
@ -3,13 +3,12 @@
|
|||
const qu = require('../utils/q');
|
||||
const slugify = require('../utils/slugify');
|
||||
|
||||
function scrapeAll(scenes, site) {
|
||||
function scrapeAll(scenes) {
|
||||
return scenes.map(({ query }) => {
|
||||
const release = {};
|
||||
const pathname = query.url('.title a');
|
||||
|
||||
release.entryId = pathname.match(/\/scene\/(\d+)/)[1];
|
||||
release.url = `${site.url}${pathname}`;
|
||||
release.url = query.url('.title a');
|
||||
release.entryId = new URL(release.url).pathname.match(/\/scene\/(\d+)/)[1];
|
||||
|
||||
release.title = query.cnt('.title a');
|
||||
release.description = query.cnt('.description');
|
||||
|
|
@ -17,7 +16,8 @@ function scrapeAll(scenes, site) {
|
|||
release.date = query.date('.date', 'MMM DD, YYYY');
|
||||
release.actors = query.cnts('.models a.model');
|
||||
|
||||
release.poster = query.q('img.poster');
|
||||
release.poster = query.img('img.poster');
|
||||
release.teaser = { src: query.video('.teaser video') };
|
||||
|
||||
release.stars = query.number('.rating');
|
||||
release.likes = query.number('.likes');
|
||||
|
|
@ -27,16 +27,16 @@ function scrapeAll(scenes, site) {
|
|||
});
|
||||
}
|
||||
|
||||
function scrapeScene({ query }) {
|
||||
function scrapeScene({ query }, url) {
|
||||
const release = {};
|
||||
|
||||
release.entryId = new URL(url).pathname.match(/\/scene\/(\d+)/)[1];
|
||||
|
||||
release.title = query.cnt('h3.title');
|
||||
release.description = query.cnt('p.description');
|
||||
|
||||
[release.poster, ...release.photos] = query.imgs('.preview-thumb');
|
||||
|
||||
const trailer = query.video('.trailer video');
|
||||
release.trailer = { src: trailer };
|
||||
release.trailer = { src: query.video('.trailer video') };
|
||||
|
||||
console.log(release);
|
||||
return release;
|
||||
|
|
@ -73,7 +73,7 @@ async function fetchScene(url, channel) {
|
|||
const res = await qu.get(url);
|
||||
|
||||
if (res.ok) {
|
||||
return scrapeScene(res.item, channel);
|
||||
return scrapeScene(res.item, url, channel);
|
||||
}
|
||||
|
||||
return res.status;
|
||||
|
|
|
|||
|
|
@ -140,10 +140,18 @@ function styles(context, selector, styleAttr) {
|
|||
return elStyles;
|
||||
}
|
||||
|
||||
function number(context, selector, attr = true) {
|
||||
function number(context, selector, match = /\d+/, attr = 'textContent') {
|
||||
const value = q(context, selector, attr);
|
||||
|
||||
return value ? Number(value) : null;
|
||||
if (value && match) {
|
||||
return Number(value.match(match)?.[0]);
|
||||
}
|
||||
|
||||
if (value) {
|
||||
return Number(value);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function meta(context, selector, attrArg = 'content', applyTrim = true) {
|
||||
|
|
@ -280,6 +288,8 @@ const quFuncs = {
|
|||
date,
|
||||
dur: duration,
|
||||
duration,
|
||||
element: q,
|
||||
el: q,
|
||||
exists,
|
||||
image,
|
||||
images,
|
||||
|
|
|
|||
Loading…
Reference in New Issue