Merge branch 'experimental'

This commit is contained in:
DebaucheryLibrarian 2020-07-17 03:41:20 +02:00
commit 0b57ebb10f
27 changed files with 222 additions and 223 deletions

View File

@ -52,6 +52,7 @@ async function mounted() {
'ebony',
'latina',
'caucasian',
'indian',
'natural-boobs',
'fake-boobs',
'blonde',

174
package-lock.json generated
View File

@ -4614,6 +4614,15 @@
"integrity": "sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg==",
"dev": true
},
"fluent-ffmpeg": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/fluent-ffmpeg/-/fluent-ffmpeg-2.1.2.tgz",
"integrity": "sha1-yVLeIkD4EuvaCqgAbXd27irPfXQ=",
"requires": {
"async": ">=0.2.9",
"which": "^1.1.1"
}
},
"flush-write-stream": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/flush-write-stream/-/flush-write-stream-1.1.1.tgz",
@ -5721,28 +5730,12 @@
"minimalistic-assert": "^1.0.1"
}
},
"hashish": {
"version": "0.0.4",
"resolved": "https://registry.npmjs.org/hashish/-/hashish-0.0.4.tgz",
"integrity": "sha1-bWC8b/r3Ebav1g5CbQd5iAFOZVQ=",
"requires": {
"traverse": ">=0.2.4"
}
},
"he": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
"integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==",
"dev": true
},
"hh-mm-ss": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/hh-mm-ss/-/hh-mm-ss-1.2.0.tgz",
"integrity": "sha512-f4I9Hz1dLpX/3mrEs7yq30+FiuO3tt5NWAqAGeBTaoeoBfB8vhcQ3BphuDc5DjZb/K809agqrAaFlP0jhEU/8w==",
"requires": {
"zero-fill": "^2.2.3"
}
},
"hmac-drbg": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/hmac-drbg/-/hmac-drbg-1.0.1.tgz",
@ -5845,11 +5838,6 @@
"debug": "^3.1.0"
}
},
"human-signals": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/human-signals/-/human-signals-1.1.1.tgz",
"integrity": "sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw=="
},
"iconv-lite": {
"version": "0.5.1",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.5.1.tgz",
@ -7168,11 +7156,6 @@
}
}
},
"merge-stream": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
"integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="
},
"mersenne-twister": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/mersenne-twister/-/mersenne-twister-1.1.0.tgz",
@ -10456,14 +10439,6 @@
"mixme": "^0.3.1"
}
},
"streamify": {
"version": "0.2.9",
"resolved": "https://registry.npmjs.org/streamify/-/streamify-0.2.9.tgz",
"integrity": "sha512-8pUxeLEef9UO1FxtTt5iikAiyzGI4SZRnGuJ3sz8axZ5Xk+/7ezEV5kuJQsMEFxw7AKYw3xp0Ow+20mmSaJbQQ==",
"requires": {
"hashish": "~0.0.4"
}
},
"strict-uri-encode": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz",
@ -10542,11 +10517,6 @@
"integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=",
"dev": true
},
"strip-final-newline": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz",
"integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA=="
},
"strip-indent": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-1.0.1.tgz",
@ -10966,11 +10936,6 @@
"punycode": "^2.1.1"
}
},
"traverse": {
"version": "0.6.6",
"resolved": "https://registry.npmjs.org/traverse/-/traverse-0.6.6.tgz",
"integrity": "sha1-y99WD9e5r2MlAv7UD5GMFX6pcTc="
},
"trim-newlines": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz",
@ -12110,127 +12075,6 @@
"camelcase": "^5.0.0",
"decamelize": "^1.2.0"
}
},
"youtube-dl": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/youtube-dl/-/youtube-dl-3.0.2.tgz",
"integrity": "sha512-LFFfpsYbRLpqKsnb4gzbnyN7fm190tJw3gJVSvfoEfnb/xYIPNT6i9G3jdzPDp/U5cwB3OSq63nUa7rUwxXAGA==",
"requires": {
"debug": "~4.1.1",
"execa": "~3.2.0",
"hh-mm-ss": "~1.2.0",
"mkdirp": "~0.5.1",
"request": "~2.88.0",
"streamify": "~0.2.9",
"universalify": "~0.1.2"
},
"dependencies": {
"cross-spawn": {
"version": "7.0.3",
"resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
"integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
"requires": {
"path-key": "^3.1.0",
"shebang-command": "^2.0.0",
"which": "^2.0.1"
}
},
"debug": {
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
"integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
"requires": {
"ms": "^2.1.1"
}
},
"execa": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/execa/-/execa-3.2.0.tgz",
"integrity": "sha512-kJJfVbI/lZE1PZYDI5VPxp8zXPO9rtxOkhpZ0jMKha56AI9y2gGVC6bkukStQf0ka5Rh15BA5m7cCCH4jmHqkw==",
"requires": {
"cross-spawn": "^7.0.0",
"get-stream": "^5.0.0",
"human-signals": "^1.1.1",
"is-stream": "^2.0.0",
"merge-stream": "^2.0.0",
"npm-run-path": "^4.0.0",
"onetime": "^5.1.0",
"p-finally": "^2.0.0",
"signal-exit": "^3.0.2",
"strip-final-newline": "^2.0.0"
}
},
"get-stream": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz",
"integrity": "sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==",
"requires": {
"pump": "^3.0.0"
}
},
"is-stream": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz",
"integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw=="
},
"mimic-fn": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz",
"integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg=="
},
"npm-run-path": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz",
"integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==",
"requires": {
"path-key": "^3.0.0"
}
},
"onetime": {
"version": "5.1.0",
"resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.0.tgz",
"integrity": "sha512-5NcSkPHhwTVFIQN+TUqXoS5+dlElHXdpAWu9I0HP20YOtIi+aZ0Ct82jdlILDxjLEAWwvm+qj1m6aEtsDVmm6Q==",
"requires": {
"mimic-fn": "^2.1.0"
}
},
"p-finally": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/p-finally/-/p-finally-2.0.1.tgz",
"integrity": "sha512-vpm09aKwq6H9phqRQzecoDpD8TmVyGw70qmWlyq5onxY7tqyTTFVvxMykxQSQKILBSFlbXpypIw2T1Ml7+DDtw=="
},
"path-key": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
"integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="
},
"shebang-command": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
"integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
"requires": {
"shebang-regex": "^3.0.0"
}
},
"shebang-regex": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
"integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="
},
"which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
"integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
"requires": {
"isexe": "^2.0.0"
}
}
}
},
"zero-fill": {
"version": "2.2.3",
"resolved": "https://registry.npmjs.org/zero-fill/-/zero-fill-2.2.3.tgz",
"integrity": "sha1-o97wa6XjmuZEhQu0yirUEStIVek="
}
}
}

View File

@ -91,6 +91,7 @@
"express-react-views": "^0.11.0",
"face-api.js": "^0.21.0",
"file-type": "^14.1.4",
"fluent-ffmpeg": "^2.1.2",
"fs-extra": "^7.0.1",
"graphile-utils": "^4.5.6",
"iconv-lite": "^0.5.1",
@ -124,7 +125,6 @@
"vuex": "^3.1.2",
"winston": "^3.2.1",
"winston-daily-rotate-file": "^4.4.2",
"yargs": "^13.3.0",
"youtube-dl": "^3.0.2"
"yargs": "^13.3.0"
}
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.7 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 253 KiB

After

Width:  |  Height:  |  Size: 792 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 7.3 KiB

After

Width:  |  Height:  |  Size: 9.1 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 40 KiB

After

Width:  |  Height:  |  Size: 44 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.3 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 356 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.1 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 46 KiB

After

Width:  |  Height:  |  Size: 36 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 47 KiB

After

Width:  |  Height:  |  Size: 37 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 29 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 28 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.6 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 14 KiB

View File

@ -514,6 +514,10 @@ const tags = [
name: 'humiliation',
slug: 'humiliation',
},
{
name: 'indian',
slug: 'indian',
},
{
name: 'innie pussy',
slug: 'innie-pussy',

View File

@ -594,7 +594,7 @@ const tagPosters = [
['atogm', 0, 'Alysa Gap and Logan in "Anal Buffet 4" for Evil Angel'],
['bdsm', 0, 'Dani Daniels in "The Traning of Dani Daniels, Day 2" for The Training of O at Kink'],
['behind-the-scenes', 0, 'Janice Griffith in "Day With A Pornstar: Janice" for Brazzers'],
['blonde', 0, 'Anikka Albrite for Digital Playground'],
['blonde', 0, 'Anikka Albrite in "Black Owned 4" for Jules Jordan'],
['blowbang', 0, 'Lacy Lennon in "Lacy Lennon\'s First Blowbang" for HardX'],
['blowjob', 0, 'Adriana Chechik in "The Dinner Party" for Real Wife Stories (Brazzers)'],
['brunette', 0, 'Nicole Black in GIO971 for LegalPorno'],
@ -615,11 +615,12 @@ const tagPosters = [
['ebony', 1, 'Ana Foxxx in "DP Me 4" for HardX'],
['facefucking', 2, 'Jynx Maze for Throated'],
['facial', 0, 'Brooklyn Gray in "All About Ass 4" for Evil Angel'],
['fake-boobs', 1, 'Lela Star in "Thick" for Jules Jordan'],
['fake-boobs', 2, 'Gia Milana in "Hot Anal Latina" for HardX'],
['family', 0, 'Teanna Trump in "A Family Appear: Part One" for Brazzers'],
['femdom', 0, 'Alina Li in "Asian Domination… She Holds Jules Jordan\'s Cock Hostage!" for Jules Jordan'],
['gangbang', 5, 'Carter Cruise\'s first gangbang in "Slut Puppies 9" for Jules Jordan'],
['gaping', 1, 'Vina Sky in "Vina Sky Does Anal" for HardX'],
['indian', 0, 'Resha in "Casting Resha" for Watch 4 Beauty'],
['interracial', 0, 'Jaye Summers and Prince Yahshua in "Platinum Pussy 3" for Jules Jordan'],
['latina', 1, 'Jynx Maze in "Big Anal Asses 2" for HardX'],
['lesbian', 0, 'Jenna Sativa and Alina Lopez in "Opposites Attract" for Girl Girl'],
@ -699,6 +700,8 @@ const tagPhotos = [
['facial', 'poster', 'Jynx Maze'],
['facefucking', 3, 'Adriana Chechik in "Performing Magic Butt Tricks With Jules Jordan. What Will Disappear In Her Ass?" for Jules Jordan'],
['facefucking', 1, 'Carrie for Young Throats'],
['fake-boobs', 1, 'Lela Star in "Thick" for Jules Jordan'],
['fake-boobs', 3, 'Ashly Anderson for Passion HD'],
// ['fake-boobs', 0, 'Marsha May in "Once You Go Black 7" for Jules Jordan'],
['gangbang', 'poster', 'Kristen Scott in "Interracial Gangbang!" for Jules Jordan'],
['gangbang', 0, '"4 On 1 Gangbangs" for Doghouse Digital'],

View File

@ -134,7 +134,7 @@ function toBaseActors(actorsOrNames, release) {
});
}
function curateActor(actor, withDetails = false) {
function curateActor(actor, withDetails = false, isProfile = false) {
if (!actor) {
return null;
}
@ -174,7 +174,7 @@ function curateActor(actor, withDetails = false) {
hasPiercings: actor.has_piercings,
tattoos: actor.tattoos,
piercings: actor.piercings,
description: actor.description,
...(isProfile && { description: actor.description }),
placeOfBirth: actor.birth_country && {
country: {
alpha2: actor.birth_country.alpha2,
@ -201,6 +201,7 @@ function curateActor(actor, withDetails = false) {
size: actor.avatar.size,
source: actor.avatar.source,
},
...(actor.profiles && { profiles: actor.profiles?.map(profile => curateActor(profile, true, true)) }),
}),
};
@ -504,12 +505,14 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
return await [].concat(source).reduce(async (outcome, scraperSlug) => outcome.catch(async () => {
try {
const scraper = scrapers[scraperSlug];
const entity = entitiesBySlug[scraperSlug] || null;
const context = {
...entitiesBySlug[scraperSlug],
...entity,
// legacy
site: entitiesBySlug[scraperSlug] || null,
network: entitiesBySlug[scraperSlug] || null,
entity: entitiesBySlug[scraperSlug] || null,
site: entity,
network: entity?.parent,
entity,
scraper: scraperSlug,
};
@ -547,7 +550,7 @@ async function scrapeProfiles(actor, sources, entitiesBySlug, existingProfilesBy
return await curateProfile({
...actor,
...profile,
...context,
entity,
update: existingProfile?.id || false,
});
} catch (error) {
@ -715,7 +718,8 @@ async function fetchActor(actorId) {
row_to_json(actor_alias) as alias,
row_to_json(birth_country) as birth_country,
row_to_json(residence_country) as residence_country,
row_to_json(media) as avatar
row_to_json(media) as avatar,
json_agg(actors_profiles) as profiles
`))
.modify((queryBuilder) => {
if (Number.isNaN(Number(actorId))) {
@ -726,10 +730,12 @@ async function fetchActor(actorId) {
queryBuilder.where('actors.id', actorId);
})
.leftJoin('actors as actor_alias', 'actor_alias.id', 'actors.alias_for')
.leftJoin('actors_profiles', 'actors.id', 'actors_profiles.actor_id')
.leftJoin('entities', 'entities.id', 'actors.entity_id')
.leftJoin('countries as birth_country', 'birth_country.alpha2', 'actors.birth_country_alpha2')
.leftJoin('countries as residence_country', 'residence_country.alpha2', 'actors.residence_country_alpha2')
.leftJoin('media', 'media.id', 'actors.avatar_media_id')
.groupBy('actors.id', 'entities.id', 'actor_alias.id', 'birth_country.alpha2', 'residence_country.alpha2', 'media.id')
.first();
return curateActor(actor, true);

View File

@ -10,7 +10,7 @@ const stream = require('stream');
const nanoid = require('nanoid/non-secure');
const mime = require('mime');
// const fileType = require('file-type');
const youtubeDl = require('youtube-dl');
const ffmpeg = require('fluent-ffmpeg');
const sharp = require('sharp');
const blake2 = require('blake2');
@ -418,22 +418,21 @@ async function fetchHttpSource(source, tempFileTarget, hashStream) {
};
}
async function fetchStreamSource(source, tempFileTarget, hashStream) {
const video = youtubeDl(source.stream);
async function fetchStreamSource(source, tempFileTarget, tempFilePath, hashStream) {
const meta = { mimetype: 'video/mp4' };
video.on('info', (info) => {
console.log(info);
logger.verbose(`Starting fetching stream from ${source.stream}`);
});
video.on('end', (info) => {
console.log(info);
logger.verbose(`Finished fetching stream from ${source.stream}`);
});
const video = ffmpeg(source.stream)
.format('mp4')
.outputOptions(['-movflags frag_keyframe+empty_moov'])
.on('start', cmd => logger.verbose(`Fetching stream from ${source.stream} with "${cmd}"`))
.on('error', error => logger.error(`Failed to fetch stream from ${source.stream}: ${error.message}`))
.pipe();
await pipeline(video, hashStream, tempFileTarget);
return { mimetype: null };
logger.verbose(`Finished fetching stream from ${source.stream}`);
return meta;
}
async function fetchSource(source, baseMedia) {
@ -457,7 +456,7 @@ async function fetchSource(source, baseMedia) {
});
const { mimetype } = source.stream
? await fetchStreamSource(source, tempFileTarget, hashStream)
? await fetchStreamSource(source, tempFileTarget, tempFilePath, hashStream)
: await fetchHttpSource(source, tempFileTarget, hashStream);
hasher.end();

View File

@ -7,8 +7,6 @@ function curateRelease(release, withMedia = false) {
return null;
}
const network = release.site_network || release.network;
return {
id: release.id,
entryId: release.entry_id,
@ -18,22 +16,22 @@ function curateRelease(release, withMedia = false) {
date: release.date,
description: release.description,
duration: release.duration,
site: release.site && {
id: release.site.id,
name: release.site.name,
slug: release.site.slug,
entity: release.entity && {
id: release.entity.id,
name: release.entity.name,
slug: release.entity.slug,
parent: release.entity.parent && {
id: release.entity.parent.id,
name: release.entity.parent.name,
slug: release.entity.parent.slug,
},
network: network && {
id: network.id,
name: network.name,
slug: network.slug,
},
actors: (release.actors || []).map(actor => ({
id: actor.id,
name: actor.name,
slug: actor.slug,
gender: actor.gender,
networkId: actor.network_id,
entityId: actor.entity_id,
aliasFor: actor.alias_for,
})),
tags: (release.tags || []).map(tag => ({
@ -67,23 +65,21 @@ function withRelations(queryBuilder, withMedia = false, type = 'scene') {
queryBuilder
.select(knex.raw(`
releases.id, releases.entry_id, releases.shoot_id, releases.title, releases.url, releases.date, releases.description, releases.duration, releases.created_at,
row_to_json(sites) as site,
row_to_json(networks) as network,
row_to_json(site_networks) as site_network,
row_to_json(entities) as entity,
row_to_json(parents) as parent,
COALESCE(json_agg(DISTINCT actors) FILTER (WHERE actors.id IS NOT NULL), '[]') as actors,
COALESCE(json_agg(DISTINCT tags) FILTER (WHERE tags.id IS NOT NULL), '[]') as tags
`))
.where('type', type)
.leftJoin('sites', 'sites.id', 'releases.site_id')
.leftJoin('networks', 'networks.id', 'releases.network_id')
.leftJoin('networks as site_networks', 'site_networks.id', 'sites.network_id')
.where('releases.type', type)
.leftJoin('entities', 'entities.id', 'releases.entity_id')
.leftJoin('entities as parents', 'parents.id', 'entities.parent_id')
.leftJoin('releases_actors', 'releases_actors.release_id', 'releases.id')
.leftJoin('actors', 'actors.id', 'releases_actors.actor_id')
.leftJoin('releases_tags', 'releases_tags.release_id', 'releases.id')
.leftJoin('tags', 'tags.id', 'releases_tags.tag_id')
.groupBy(knex.raw(`
releases.id, releases.entry_id, releases.shoot_id, releases.title, releases.url, releases.date, releases.description, releases.duration, releases.created_at,
sites.id, networks.id, site_networks.id
entities.id, parents.id
`));
if (withMedia) {

View File

@ -0,0 +1,136 @@
'use strict';
const qu = require('../utils/q');
const slugify = require('../utils/slugify');
function scrapeAll(scenes, channel) {
return scenes.map(({ query }) => {
const release = {};
release.url = query.url('.scene-update-details, .feature-update-details', 'href', { origin: channel.url });
release.entryId = new URL(release.url).pathname.match(/\/(\d+)/)[1];
release.title = query.q('.scene-img-wrapper img', 'alt').replace(/\s*image$/i, '');
release.date = query.date('.scene-update-stats span, .feature-update-details span', 'MMM DD, YYYY');
release.actors = query.cnt('.scene-update-details h3, .feature-update-details h2')?.split(/\s*\|\s*/).map(actor => actor.trim());
const poster = query.img('.scene-img-wrapper img');
release.poster = [
poster.replace(/\/res\/\d+/, '/res/1920'),
poster.replace(/\/res\/\d+/, '/res/1600'),
poster,
];
release.trailer = { src: query.video('.scene-img-wrapper source') };
return release;
});
}
async function scrapeScene({ query, html }, url) {
const release = {};
release.entryId = new URL(url).pathname.match(/\/(\d+)/)[1];
release.title = query.cnt('.scene-page .description');
release.date = query.date('.release-date:first-child', 'MMM DD, YYYY', /\w{3} \d{2}, \d{4}/);
release.duration = query.number('.release-date:last-child') * 60;
release.actors = query.all('.video-performer').map((el) => {
const avatar = qu.query.img(el, 'img', 'data-bgsrc');
return {
name: qu.query.cnt(el, 'span'),
avatar: [
avatar.replace(/\/actor\/\d+/, '/actor/1600'),
avatar,
],
};
});
release.tags = query.cnts('.tags a');
release.poster = query.url('link[rel="image_src"]') || query.meta('property="og:image"');
release.photos = query.imgs('#dv_frames a > img').map(photo => [
photo.replace(/(\/p\/\d+\/)\d+/, (match, path) => `${path}1920`),
photo.replace(/(\/p\/\d+\/)\d+/, (match, path) => `${path}1600`),
photo,
]);
const trailerId = html.match(/item: (\d+),/)?.[1];
if (trailerId) {
const trailerUrl = `https://www.adultempire.com/videoEmbed/${trailerId}?type=preview`;
const trailerRes = await qu.get(trailerUrl);
if (trailerRes.ok) {
const stream = trailerRes.item.query.video();
release.trailer = { stream };
}
}
// console.log(release);
return release;
}
function scrapeProfile({ query, el }, actorName, entity, include) {
const profile = {};
profile.description = query.cnt('.bio-text');
profile.birthPlace = query.cnt('.birth-place span');
profile.avatar = query.img('.actor-photo img');
if (include.releases) {
return scrapeAll(qu.initAll(el, '.scene'));
}
console.log(profile);
return profile;
}
async function fetchLatest(channel, page = 1) {
const url = `${channel.url}/tour?page=${page}`;
const res = await qu.getAll(url, '.scene-update', null, {
// invalid certificate
rejectUnauthorized: false,
});
if (res.ok) {
return scrapeAll(res.items, channel);
}
return res.status;
}
async function fetchScene(url, channel) {
const res = await qu.get(url, null, null, {
// invalid certificate
rejectUnauthorized: false,
});
if (res.ok) {
return scrapeScene(res.item, url, channel);
}
return res.status;
}
async function fetchProfile(actorName, entity, include) {
const url = `${entity.url}/actors/${slugify(actorName, '_')}`;
const res = await qu.get(url);
if (res.ok) {
return scrapeProfile(res.item, actorName, entity, include);
}
return res.status;
}
module.exports = {
fetchLatest,
fetchScene,
fetchProfile,
};

View File

@ -3,13 +3,12 @@
const qu = require('../utils/q');
const slugify = require('../utils/slugify');
function scrapeAll(scenes, site) {
function scrapeAll(scenes) {
return scenes.map(({ query }) => {
const release = {};
const pathname = query.url('.title a');
release.entryId = pathname.match(/\/scene\/(\d+)/)[1];
release.url = `${site.url}${pathname}`;
release.url = query.url('.title a');
release.entryId = new URL(release.url).pathname.match(/\/scene\/(\d+)/)[1];
release.title = query.cnt('.title a');
release.description = query.cnt('.description');
@ -17,7 +16,8 @@ function scrapeAll(scenes, site) {
release.date = query.date('.date', 'MMM DD, YYYY');
release.actors = query.cnts('.models a.model');
release.poster = query.q('img.poster');
release.poster = query.img('img.poster');
release.teaser = { src: query.video('.teaser video') };
release.stars = query.number('.rating');
release.likes = query.number('.likes');
@ -27,16 +27,16 @@ function scrapeAll(scenes, site) {
});
}
function scrapeScene({ query }) {
function scrapeScene({ query }, url) {
const release = {};
release.entryId = new URL(url).pathname.match(/\/scene\/(\d+)/)[1];
release.title = query.cnt('h3.title');
release.description = query.cnt('p.description');
[release.poster, ...release.photos] = query.imgs('.preview-thumb');
const trailer = query.video('.trailer video');
release.trailer = { src: trailer };
release.trailer = { src: query.video('.trailer video') };
console.log(release);
return release;
@ -73,7 +73,7 @@ async function fetchScene(url, channel) {
const res = await qu.get(url);
if (res.ok) {
return scrapeScene(res.item, channel);
return scrapeScene(res.item, url, channel);
}
return res.status;

View File

@ -140,10 +140,18 @@ function styles(context, selector, styleAttr) {
return elStyles;
}
function number(context, selector, attr = true) {
function number(context, selector, match = /\d+/, attr = 'textContent') {
const value = q(context, selector, attr);
return value ? Number(value) : null;
if (value && match) {
return Number(value.match(match)?.[0]);
}
if (value) {
return Number(value);
}
return null;
}
function meta(context, selector, attrArg = 'content', applyTrim = true) {
@ -280,6 +288,8 @@ const quFuncs = {
date,
dur: duration,
duration,
element: q,
el: q,
exists,
image,
images,