Merge branch 'experimental'

This commit is contained in:
ThePendulum 2020-07-13 00:12:11 +02:00
commit 50e5525591
89 changed files with 235 additions and 108 deletions

View File

@ -94,7 +94,7 @@
<a <a
v-else v-else
:href="`/channel/${release.entity.slug}`" :href="`/${release.entity.type}/${release.entity.slug}`"
> >
<img <img
:src="`/img/logos/${release.entity.slug}/thumbs/network.png`" :src="`/img/logos/${release.entity.slug}/thumbs/network.png`"

View File

@ -79,7 +79,7 @@ export default {
box-shadow: 0 0 3px var(--darken); box-shadow: 0 0 3px var(--darken);
.favicon { .favicon {
height: 1rem; width: 1rem;
margin: 0 .25rem 0 0; margin: 0 .25rem 0 0;
} }

View File

@ -121,6 +121,9 @@ module.exports = {
], ],
'21sextury', '21sextury',
'julesjordan', 'julesjordan',
'bang',
'pervcity',
'kink',
'peternorth', 'peternorth',
'naughtyamerica', 'naughtyamerica',
'cherrypimps', 'cherrypimps',

View File

@ -174,7 +174,7 @@ exports.up = knex => Promise.resolve()
table.unique(['slug', 'type']); table.unique(['slug', 'type']);
table.text('alias'); table.specificType('alias', 'text[]');
table.text('url'); table.text('url');
table.text('description'); table.text('description');
@ -820,6 +820,8 @@ exports.up = knex => Promise.resolve()
WHERE WHERE
name ILIKE ('%' || search || '%') OR name ILIKE ('%' || search || '%') OR
slug ILIKE ('%' || search || '%') OR slug ILIKE ('%' || search || '%') OR
array_to_string(alias, '') ILIKE ('%' || search || '%') OR
replace(array_to_string(alias, ''), ' ', '') ILIKE ('%' || search || '%') OR
url ILIKE ('%' || search || '%') url ILIKE ('%' || search || '%')
$$ LANGUAGE SQL STABLE; $$ LANGUAGE SQL STABLE;

Binary file not shown.

Before

Width:  |  Height:  |  Size: 34 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.0 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 28 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.5 KiB

After

Width:  |  Height:  |  Size: 2.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.4 KiB

After

Width:  |  Height:  |  Size: 2.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.6 KiB

After

Width:  |  Height:  |  Size: 3.6 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.7 KiB

After

Width:  |  Height:  |  Size: 3.7 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.8 KiB

After

Width:  |  Height:  |  Size: 3.8 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.5 KiB

After

Width:  |  Height:  |  Size: 3.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.8 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.4 KiB

After

Width:  |  Height:  |  Size: 2.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 166 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 27 KiB

After

Width:  |  Height:  |  Size: 27 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 17 KiB

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 36 KiB

After

Width:  |  Height:  |  Size: 36 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.7 KiB

After

Width:  |  Height:  |  Size: 3.7 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 30 KiB

After

Width:  |  Height:  |  Size: 30 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 30 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 20 KiB

After

Width:  |  Height:  |  Size: 20 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 10 KiB

After

Width:  |  Height:  |  Size: 10 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 36 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 12 KiB

After

Width:  |  Height:  |  Size: 12 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 16 KiB

After

Width:  |  Height:  |  Size: 16 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 22 KiB

After

Width:  |  Height:  |  Size: 22 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 18 KiB

After

Width:  |  Height:  |  Size: 19 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 29 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 19 KiB

After

Width:  |  Height:  |  Size: 19 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 18 KiB

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.5 KiB

After

Width:  |  Height:  |  Size: 2.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.6 KiB

After

Width:  |  Height:  |  Size: 2.6 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.4 KiB

After

Width:  |  Height:  |  Size: 2.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.3 KiB

After

Width:  |  Height:  |  Size: 2.3 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.7 KiB

After

Width:  |  Height:  |  Size: 2.7 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.4 KiB

After

Width:  |  Height:  |  Size: 2.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.4 KiB

After

Width:  |  Height:  |  Size: 2.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.3 KiB

After

Width:  |  Height:  |  Size: 2.3 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.4 KiB

After

Width:  |  Height:  |  Size: 2.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.4 KiB

After

Width:  |  Height:  |  Size: 2.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.7 KiB

After

Width:  |  Height:  |  Size: 2.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.7 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.7 KiB

After

Width:  |  Height:  |  Size: 2.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.7 KiB

View File

Before

Width:  |  Height:  |  Size: 40 KiB

After

Width:  |  Height:  |  Size: 40 KiB

View File

Before

Width:  |  Height:  |  Size: 7.8 KiB

After

Width:  |  Height:  |  Size: 7.8 KiB

View File

Before

Width:  |  Height:  |  Size: 27 KiB

After

Width:  |  Height:  |  Size: 27 KiB

View File

Before

Width:  |  Height:  |  Size: 2.0 KiB

After

Width:  |  Height:  |  Size: 2.0 KiB

View File

Before

Width:  |  Height:  |  Size: 27 KiB

After

Width:  |  Height:  |  Size: 27 KiB

View File

Before

Width:  |  Height:  |  Size: 44 KiB

After

Width:  |  Height:  |  Size: 44 KiB

View File

Before

Width:  |  Height:  |  Size: 3.4 KiB

After

Width:  |  Height:  |  Size: 3.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 39 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 18 KiB

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 27 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 8.9 KiB

After

Width:  |  Height:  |  Size: 8.9 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 8.9 KiB

After

Width:  |  Height:  |  Size: 8.9 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 8.5 KiB

After

Width:  |  Height:  |  Size: 8.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 8.1 KiB

After

Width:  |  Height:  |  Size: 8.1 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 9.1 KiB

After

Width:  |  Height:  |  Size: 9.1 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 7.7 KiB

After

Width:  |  Height:  |  Size: 7.7 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 7.2 KiB

After

Width:  |  Height:  |  Size: 7.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 8.5 KiB

After

Width:  |  Height:  |  Size: 8.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 8.1 KiB

After

Width:  |  Height:  |  Size: 8.1 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 6.9 KiB

After

Width:  |  Height:  |  Size: 6.9 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 8.6 KiB

After

Width:  |  Height:  |  Size: 8.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 8.6 KiB

After

Width:  |  Height:  |  Size: 8.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

View File

@ -136,9 +136,13 @@ const networks = [
type: 'info', type: 'info',
}, },
{ {
slug: 'ddfnetwork', slug: 'pornworld',
name: 'DDF Network', name: 'Porn World',
url: 'https://ddfnetwork.com', alias: [
'ddf network',
'denys defrancesco',
],
url: 'https://pornworld.com',
description: 'European porn videos hub with exclusive VR, 4K and full HD XXX videos and hot sex photos of Europes finest porn star babes.', description: 'European porn videos hub with exclusive VR, 4K and full HD XXX videos and hot sex photos of Europes finest porn star babes.',
parent: 'wgcz', parent: 'wgcz',
}, },
@ -417,7 +421,7 @@ exports.seed = knex => Promise.resolve()
slug: network.slug, slug: network.slug,
name: network.name, name: network.name,
type: network.type || 'network', type: network.type || 'network',
alias: network.alias ? network.alias.join(',') : null, alias: network.alias,
url: network.url, url: network.url,
description: network.description, description: network.description,
parameters: network.parameters, parameters: network.parameters,

View File

@ -1455,7 +1455,7 @@ const sites = [
alias: ['ddfb'], alias: ['ddfb'],
url: 'https://ddfbusty.com', url: 'https://ddfbusty.com',
description: 'Gorgeous Babes with big tits and Euro pornstars with huge natural boobs filmed in Exclusive Full HD, 4K, & VR porn videos.', description: 'Gorgeous Babes with big tits and Euro pornstars with huge natural boobs filmed in Exclusive Full HD, 4K, & VR porn videos.',
parent: 'ddfnetwork', parent: 'pornworld',
}, },
{ {
slug: 'handsonhardcore', slug: 'handsonhardcore',
@ -1463,7 +1463,7 @@ const sites = [
alias: ['hoh'], alias: ['hoh'],
url: 'https://handsonhardcore.com', url: 'https://handsonhardcore.com',
description: 'Hardcore Sex & Anal Fucking Exclusive XXX Videos in VR, 4K and full HD with Hot European Pornstars', description: 'Hardcore Sex & Anal Fucking Exclusive XXX Videos in VR, 4K and full HD with Hot European Pornstars',
parent: 'ddfnetwork', parent: 'pornworld',
}, },
{ {
slug: 'houseoftaboo', slug: 'houseoftaboo',
@ -1471,7 +1471,7 @@ const sites = [
alias: ['hotb', 'hotab'], alias: ['hotb', 'hotab'],
url: 'https://houseoftaboo.com', url: 'https://houseoftaboo.com',
description: 'Exclusive BDSM Porn & Extreme Sex Videos Produced in VR, 4K and full HD with The Hottest European Fetish Pornstars', description: 'Exclusive BDSM Porn & Extreme Sex Videos Produced in VR, 4K and full HD with The Hottest European Fetish Pornstars',
parent: 'ddfnetwork', parent: 'pornworld',
}, },
{ {
slug: 'ddfnetworkvr', slug: 'ddfnetworkvr',
@ -1479,21 +1479,24 @@ const sites = [
alias: ['ddfvr'], alias: ['ddfvr'],
url: 'https://ddfnetworkvr.com', url: 'https://ddfnetworkvr.com',
description: 'VR Porn Videos shot Exclusively in 180 3D 4K Virtual Reality featuring the Hottest European & American VR Pornstar Babes', description: 'VR Porn Videos shot Exclusively in 180 3D 4K Virtual Reality featuring the Hottest European & American VR Pornstar Babes',
parent: 'ddfnetwork', parent: 'pornworld',
parameters: {
latest: 'https://ddfnetworkvr.com/home/tagfiltered/keywords/-/',
},
}, },
{ {
slug: 'eurogirlsongirls', slug: 'eurogirlsongirls',
name: 'Euro Girls on Girls', name: 'Euro Girls on Girls',
url: 'https://eurogirlsongirls.com', url: 'https://eurogirlsongirls.com',
description: 'Hot Lesbian Sex & Glamour Lesbian Porn Videos and Photos Starring Gorgeous European Pornstars in 4K and Full HD VR.', description: 'Hot Lesbian Sex & Glamour Lesbian Porn Videos and Photos Starring Gorgeous European Pornstars in 4K and Full HD VR.',
parent: 'ddfnetwork', parent: 'pornworld',
}, },
{ {
slug: '1byday', slug: '1byday',
name: '1By-Day', name: '1By-Day',
url: 'https://1by-day.com', url: 'https://1by-day.com',
description: 'Ultra Sexy Exclusive Solo Masturbation Videos in VR, 4K and full HD showcasing Glamour Babes & Intense Orgasms', description: 'Ultra Sexy Exclusive Solo Masturbation Videos in VR, 4K and full HD showcasing Glamour Babes & Intense Orgasms',
parent: 'ddfnetwork', parent: 'pornworld',
}, },
{ {
slug: 'euroteenerotica', slug: 'euroteenerotica',
@ -1501,14 +1504,14 @@ const sites = [
alias: ['ete'], alias: ['ete'],
url: 'https://euroteenerotica.com', url: 'https://euroteenerotica.com',
description: 'Teen Threesomes & Barely Legal Porn Videos in 4K, VR and FULL HD with Hot Nymphomaniac Teen Babes', description: 'Teen Threesomes & Barely Legal Porn Videos in 4K, VR and FULL HD with Hot Nymphomaniac Teen Babes',
parent: 'ddfnetwork', parent: 'pornworld',
}, },
{ {
slug: 'hotlegsandfeet', slug: 'hotlegsandfeet',
name: 'Hot Legs and Feet', name: 'Hot Legs and Feet',
url: 'https://hotlegsandfeet.com', url: 'https://hotlegsandfeet.com',
description: 'Foot Fetish & Sexy Legs Porn Videos with Hot and Sexy Euro Pornstars', description: 'Foot Fetish & Sexy Legs Porn Videos with Hot and Sexy Euro Pornstars',
parent: 'ddfnetwork', parent: 'pornworld',
}, },
{ {
slug: 'onlyblowjob', slug: 'onlyblowjob',
@ -1516,22 +1519,24 @@ const sites = [
alias: ['obj'], alias: ['obj'],
url: 'https://onlyblowjob.com', url: 'https://onlyblowjob.com',
description: 'Fantasy Blowjobs & POV Cock Sucking Videos and Photos Produced in VR, 4K and full HD featuring Sexy European Pornstars', description: 'Fantasy Blowjobs & POV Cock Sucking Videos and Photos Produced in VR, 4K and full HD featuring Sexy European Pornstars',
parent: 'ddfnetwork', parent: 'pornworld',
}, },
{ {
slug: 'fuckinhd', slug: 'fuckinhd',
name: 'Fuck in HD', name: 'Fuck in HD',
url: 'https://fuckinhd.com', url: 'https://fuckinhd.com',
description: 'HD Hardcore Sex & XXX Fantasy Porn Videos and Photos Produced in full HD featuring a Variety of Hardcore Porn Niches.', description: 'HD Hardcore Sex & XXX Fantasy Porn Videos and Photos Produced in full HD featuring a Variety of Hardcore Porn Niches.',
parent: 'ddfnetwork', parent: 'pornworld',
parameters: { native: true },
}, },
{ {
slug: 'bustylover', slug: 'bustylover',
name: 'Busty Lover', name: 'Busty Lover',
url: 'https://bustylover.com', url: 'https://bustylover.com',
parent: 'ddfnetwork', parent: 'pornworld',
parameters: { native: true }, parameters: {
latest: 'https://bustylover.com/videos/',
blockLayout: true,
},
}, },
// DIGITAL PLAYGROUND // DIGITAL PLAYGROUND
{ {
@ -7434,7 +7439,7 @@ exports.seed = knex => Promise.resolve()
slug: site.slug, slug: site.slug,
name: site.name, name: site.name,
type: site.type || 'channel', type: site.type || 'channel',
alias: (site.alias || []).join(','), alias: site.alias,
description: site.description, description: site.description,
url: site.url, url: site.url,
parameters: site.parameters, parameters: site.parameters,

View File

@ -64,9 +64,10 @@ const ethnicities = {
black: 'black', black: 'black',
caucasian: 'white', caucasian: 'white',
european: 'white', european: 'white',
hispanic: 'latina', hispanic: 'latin',
indian: 'indian', indian: 'indian',
japanese: 'japanese', japanese: 'japanese',
latin: 'latin',
latina: 'latina', latina: 'latina',
latino: 'latino', latino: 'latino',
white: 'white', white: 'white',

View File

@ -591,13 +591,20 @@ async function storeMedias(baseMedias) {
); );
const [uniqueHashMedias, existingHashMedias] = await findHashDuplicates(fetchedMedias); const [uniqueHashMedias, existingHashMedias] = await findHashDuplicates(fetchedMedias);
const newMedias = argv.force ? uniqueHashMedias.concat(existingHashMedias) : uniqueHashMedias;
const savedMedias = await Promise.map( const savedMedias = await Promise.map(
newMedias, uniqueHashMedias,
async baseMedia => storeFile(baseMedia), async baseMedia => storeFile(baseMedia),
); );
if (argv.force) {
// overwrite files in case image processing was changed
await Promise.map(
existingHashMedias,
async baseMedia => storeFile(baseMedia),
);
}
const newMediaWithEntries = savedMedias.map((media, index) => curateMediaEntry(media, index)); const newMediaWithEntries = savedMedias.map((media, index) => curateMediaEntry(media, index));
const newMediaEntries = newMediaWithEntries.filter(media => media.newEntry).map(media => media.entry); const newMediaEntries = newMediaWithEntries.filter(media => media.newEntry).map(media => media.entry);

View File

@ -2,53 +2,60 @@
const bhttp = require('bhttp'); const bhttp = require('bhttp');
const { ed, ex, exa, get } = require('../utils/q'); const qu = require('../utils/qu');
const slugify = require('../utils/slugify'); const slugify = require('../utils/slugify');
/* eslint-disable newline-per-chained-call */ function scrapeAll(scenes, site, origin) {
function scrapeAll(html, site, origin) { return scenes.map(({ query }) => {
return exa(html, '.card.m-1:not(.pornstar-card)').map(({ q, qa, qd }) => {
const release = {}; const release = {};
release.title = q('a', 'title'); release.title = query.q('.card-title a, .videoContent h4 a', 'title');
release.url = `${site?.url || origin || 'https://ddfnetwork.com'}${q('a', 'href')}`; release.url = `${site?.url || origin || 'https://pornworld.com'}${query.q('a', 'href')}`;
[release.entryId] = release.url.split('/').slice(-1); release.entryId = release.url.match(/\/(\d+)$/)[1];
release.date = qd('small[datetime]', 'YYYY-MM-DD HH:mm:ss', null, 'datetime'); release.date = query.date('small[datetime]', 'YYYY-MM-DD HH:mm:ss', null, 'datetime');
release.actors = qa('.card-subtitle a', true).filter(Boolean); release.actors = query.all('.card-subtitle a, .featuring a', true).filter(Boolean);
const duration = parseInt(q('.card-info div:nth-child(2) .card-text', true), 10) * 60; release.description = query.q('h4 + p', true);
const duration = parseInt(query.q('.card-info div:nth-child(2) .card-text', true), 10) * 60;
if (duration) release.duration = duration; if (duration) release.duration = duration;
else release.duration = query.dur('.time');
release.poster = q('img').dataset.src; release.poster = query.img();
return release; return release;
}); });
} }
async function scrapeScene(html, url, _site) { async function scrapeScene({ query }, url, _site) {
const { qu } = ex(html);
const release = {}; const release = {};
[release.entryId] = url.split('/').slice(-1); release.entryId = url.match(/\/(\d+)$/)[1];
release.title = qu.meta('itemprop=name'); release.title = query.meta('itemprop=name') || query.q('.video-title h1', true) || query.q('.about-text .story-title') || query.q('h3', true);
release.description = qu.q('.descr-box p', true); release.description = query.q('.descr-box p', true) || query.q('.about-text p:not(.story-title)', true) || query.text('.description p');
release.date = qu.date('meta[itemprop=uploadDate]', 'YYYY-MM-DD', null, 'content')
|| qu.date('.title-border:nth-child(2) p', 'MM.DD.YYYY');
release.actors = qu.all('.pornstar-card > a', 'title'); release.date = query.date('meta[itemprop=uploadDate]', 'YYYY-MM-DD', null, 'content')
release.tags = qu.all('.tags-tab .tags a', true); || query.date('.actors time', 'MMMM DD, YYYY')
|| query.date('.title-border:nth-child(2) p', 'MM.DD.YYYY')
|| query.date('.length', 'MMMM DD, YYYY', /\w+ \d{2}, \d{4}/);
release.duration = parseInt(qu.q('.icon-video-red + span', true), 10) * 60; if (query.exists('.pornstar-card > a')) release.actors = query.all('.pornstar-card > a', 'title');
release.likes = Number(qu.q('.icon-like-red + span', true)); else if (query.exists('.actors a')) release.actors = query.all('.actors a', true);
release.poster = qu.poster(); if (query.exists('.tags-tab')) release.tags = query.all('.tags-tab .tags a', true);
release.photos = qu.urls('.photo-slider-guest .card a'); else if (query.exists('.tags-box')) release.tags = query.all('.tags-box .tags li', true);
release.trailer = qu.all('source[type="video/mp4"]').map(trailer => ({ release.duration = parseInt(query.q('.icon-video-red + span', true), 10) * 60 || query.dur('.length') || null;
release.likes = Number(query.q('.icon-like-red + span', true)) || null;
release.poster = query.poster() || query.poster('dl8-video');
release.photos = query.urls('.photo-slider-guest .card a');
release.trailer = query.all('source[type="video/mp4"]').map(trailer => ({
src: trailer.src, src: trailer.src,
quality: Number(trailer.attributes.res.value), quality: Number(trailer.attributes.res?.value || trailer.attributes.quality?.value.slice(0, -1)) || null,
})); }));
return release; return release;
@ -59,9 +66,9 @@ async function fetchActorReleases(urls) {
const sources = urls.filter(url => !/ddfnetwork/.test(url)); const sources = urls.filter(url => !/ddfnetwork/.test(url));
const releases = await Promise.all(sources.map(async (url) => { const releases = await Promise.all(sources.map(async (url) => {
const { html } = await get(url); const res = await qu.getAll(url, '.card.m-1:not(.pornstar-card)');
return scrapeAll(html, null, new URL(url).origin); return res.ok ? scrapeAll(res.items, null, new URL(url).origin) : null;
})); }));
// DDF cross-releases scenes between sites, filter duplicates by entryId // DDF cross-releases scenes between sites, filter duplicates by entryId
@ -71,11 +78,9 @@ async function fetchActorReleases(urls) {
.reduce((acc, release) => ({ ...acc, [release.entryId]: release }), {})); .reduce((acc, release) => ({ ...acc, [release.entryId]: release }), {}));
} }
async function scrapeProfile(html, _url, actorName) { async function scrapeProfile({ query }, _url, actorName) {
const { qu } = ex(html); const keys = query.all('.about-title', true).map(key => slugify(key, '_'));
const values = query.all('.about-info').map((el) => {
const keys = qu.all('.about-title', true).map(key => slugify(key, '_'));
const values = qu.all('.about-info').map((el) => {
if (el.children.length > 0) { if (el.children.length > 0) {
return Array.from(el.children, child => child.textContent.trim()).join(', '); return Array.from(el.children, child => child.textContent.trim()).join(', ');
} }
@ -96,8 +101,8 @@ async function scrapeProfile(html, _url, actorName) {
name: actorName, name: actorName,
}; };
profile.description = qu.q('.description-box', true); profile.description = query.q('.description-box', true);
profile.birthdate = ed(bio.birthday, 'MMMM DD, YYYY'); profile.birthdate = qu.extractDate(bio.birthday, 'MMMM DD, YYYY');
if (bio.nationality) profile.nationality = bio.nationality; if (bio.nationality) profile.nationality = bio.nationality;
@ -118,34 +123,37 @@ async function scrapeProfile(html, _url, actorName) {
if (bio.shoe_size) profile.shoes = Number(bio.shoe_size.split('|')[1]); if (bio.shoe_size) profile.shoes = Number(bio.shoe_size.split('|')[1]);
const avatarEl = qu.q('.pornstar-details .card-img-top'); const avatarEl = query.q('.pornstar-details .card-img-top');
if (avatarEl && avatarEl.dataset.src.match('^//')) profile.avatar = `https:${avatarEl.dataset.src}`; if (avatarEl && avatarEl.dataset.src.match('^//')) profile.avatar = `https:${avatarEl.dataset.src}`;
profile.releases = await fetchActorReleases(qu.urls('.find-me-tab li a')); profile.releases = await fetchActorReleases(query.urls('.find-me-tab li a'));
return profile; return profile;
} }
async function fetchLatest(site, page = 1) { async function fetchLatest(channel, page = 1) {
/* ddfnetwork.com redirects to pornworld.com
const url = site.parameters?.native const url = site.parameters?.native
? `${site.url}/videos/search/latest/ever/allsite/-/${page}` ? `${site.url}/videos/search/latest/ever/allsite/-/${page}`
: `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`; : `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`;
*/
const res = await bhttp.get(url); const url = channel.parameters?.latest || `${channel.url}/videos/search/latest/ever/allsite/-/${page}`;
const res = await qu.getAll(url, '.card.m-1:not(.pornstar-card), .allVideos .videoBlock');
if (res.statusCode === 200) { if (res.ok) {
return scrapeAll(res.body.toString(), site); return scrapeAll(res.items, channel);
} }
return res.statusCode; return res.status;
} }
async function fetchScene(url, site) { async function fetchScene(url, site) {
// DDF's main site moved to Porn World // DDF's main site moved to Porn World
// const res = await bhttp.get(`https://ddfnetwork.com${new URL(url).pathname}`); // const res = await bhttp.get(`https://ddfnetwork.com${new URL(url).pathname}`);
const res = await bhttp.get(url); const res = await qu.get(url, '.content, #content, .taspVideoPage');
return scrapeScene(res.body.toString(), url, site); return res.ok ? scrapeScene(res.item, url, site) : res.status;
} }
async function fetchProfile(actorName) { async function fetchProfile(actorName) {
@ -155,7 +163,7 @@ async function fetchProfile(actorName) {
word: actorName, word: actorName,
}, },
{ {
decodeJSON: true, decodeJSON: false,
headers: { headers: {
'x-requested-with': 'XMLHttpRequest', 'x-requested-with': 'XMLHttpRequest',
}, },

View File

@ -1,54 +1,54 @@
'use strict'; 'use strict';
const { get, getAll } = require('../utils/qu'); const qu = require('../utils/qu');
function scrapeLatest(scenes) { function scrapeAll(scenes) {
return scenes.map(({ qu }) => { return scenes.map(({ query }) => {
const release = {}; const release = {};
const href = qu.url('.shoot-thumb-title a'); const href = query.url('.shoot-thumb-title a');
release.url = `https://kink.com${href}`; release.url = `https://kink.com${href}`;
release.shootId = href.split('/').slice(-1)[0]; release.shootId = href.split('/').slice(-1)[0];
release.entryId = release.shootId; release.entryId = release.shootId;
release.title = qu.q('.shoot-thumb-title a', true); release.title = query.q('.shoot-thumb-title a', true);
release.date = qu.date('.date', 'MMM DD, YYYY'); release.date = query.date('.date', 'MMM DD, YYYY');
release.actors = qu.all('.shoot-thumb-models a', true); release.actors = query.all('.shoot-thumb-models a', true);
release.stars = qu.q('.average-rating', 'data-rating') / 10; release.stars = query.q('.average-rating', 'data-rating') / 10;
release.poster = qu.img('.adimage'); release.poster = query.img('.adimage');
release.photos = qu.imgs('.rollover .roll-image', 'data-imagesrc').map(photo => [ release.photos = query.imgs('.rollover .roll-image', 'data-imagesrc').map(photo => [
photo.replace('410/', '830/'), photo.replace('410/', '830/'),
photo, photo,
]); ]);
release.duration = qu.dur('.video span'); release.duration = query.dur('.video span');
return release; return release;
}); });
} }
async function scrapeScene({ qu }, url) { async function scrapeScene({ query }, url) {
const release = { url }; const release = { url };
release.shootId = new URL(url).pathname.split('/')[2]; release.shootId = new URL(url).pathname.split('/')[2];
release.entryId = release.shootId; release.entryId = release.shootId;
release.title = qu.q('.shoot-title span.favorite-button', 'data-title'); release.title = query.q('.shoot-title span.favorite-button', 'data-title');
release.description = qu.q('.description-text', true); release.description = query.q('.description-text', true);
release.date = qu.date('.shoot-date', 'MMMM DD, YYYY'); release.date = query.date('.shoot-date', 'MMMM DD, YYYY');
release.actors = qu.all('.names a', true).map(actor => actor.replace(/,\s*/, '')); release.actors = query.all('.names a', true).map(actor => actor.replace(/,\s*/, ''));
release.director = qu.q('.director-name', true); release.director = query.q('.director-name', true);
release.photos = qu.imgs('.gallery .thumb img', 'data-image-file'); release.photos = query.imgs('.gallery .thumb img', 'data-image-file');
release.poster = qu.poster(); release.poster = query.poster();
release.tags = qu.all('.tag-list a[href*="/tag"]', true).map(tag => tag.replace(/,\s*/, '')); release.tags = query.all('.tag-list a[href*="/tag"]', true).map(tag => tag.replace(/,\s*/, ''));
const trailer = qu.q('.player span[data-type="trailer-src"]', 'data-url'); const trailer = query.q('.player span[data-type="trailer-src"]', 'data-url');
release.trailer = [ release.trailer = [
{ {
@ -69,23 +69,77 @@ async function scrapeScene({ qu }, url) {
}, },
]; ];
release.channel = qu.url('.shoot-logo a').split('/').slice(-1)[0]; release.channel = query.url('.shoot-logo a').split('/').slice(-1)[0];
return release; return release;
} }
async function fetchLatest(site, page = 1) { async function fetchActorReleases(actorUrl, page = 1, accReleases = []) {
const res = await getAll(`${site.url}/latest/page/${page}`, '.shoot-list .shoot'); const res = await qu.get(`${actorUrl}?page=${page}`);
if (res.ok) { if (res.ok) {
return scrapeLatest(res.items, site); const releases = scrapeAll(qu.initAll(res.item.el, '.shoot-list .shoot'));
const hasNextPage = res.item.query.exists('.paginated-nav li:last-child:not(.disabled)');
if (hasNextPage) {
return fetchActorReleases(actorUrl, page + 1, accReleases.concat(releases));
}
return accReleases.concat(releases);
}
return accReleases;
}
async function scrapeProfile({ query }, actorUrl, include) {
const profile = {};
profile.description = query.q('.bio #expand-text', true);
const tags = query.all('.bio-tags a', true);
if (tags.includes('brunette') || tags.includes('brunet')) profile.hairColor = 'brown';
if (tags.includes('blonde') || tags.includes('blond')) profile.hairColor = 'blonde';
if (tags.includes('black hair')) profile.hairColor = 'black';
if (tags.includes('redhead')) profile.hairColor = 'red';
if (tags.includes('natural boobs')) profile.naturalBoobs = true;
if (tags.includes('fake boobs')) profile.naturalBoobs = false;
if (tags.includes('white')) profile.ethnicity = 'white';
if (tags.includes('latin')) profile.ethnicity = 'latin';
if (tags.includes('Black')) profile.ethnicity = 'black';
if (tags.includes('pierced nipples')) profile.hasPiercings = true;
if (tags.includes('tattoo')) profile.hasTattoos = true;
if (tags.includes('foreskin')) profile.hasForeskin = true;
if ((tags.includes('big dick') || tags.includes('foreskin'))
&& (tags.includes('fake boobs') || tags.includes('big tits'))) profile.gender = 'transsexual';
profile.avatar = query.img('.bio-slider-img, .bio-img:not([src*="Missing"])');
profile.social = query.urls('a.social-link');
if (include.releases) {
profile.releases = await fetchActorReleases(actorUrl);
}
return profile;
}
async function fetchLatest(site, page = 1) {
const res = await qu.getAll(`${site.url}/latest/page/${page}`, '.shoot-list .shoot');
if (res.ok) {
return scrapeAll(res.items, site);
} }
return res.status; return res.status;
} }
async function fetchScene(url, site) { async function fetchScene(url, site) {
const res = await get(url); const res = await qu.get(url);
if (res.ok) { if (res.ok) {
return scrapeScene(res.item, url, site); return scrapeScene(res.item, url, site);
@ -94,7 +148,32 @@ async function fetchScene(url, site) {
return res.status; return res.status;
} }
async function fetchProfile(actorName, entity, include) {
const searchRes = await qu.getAll(`https://kink.com/search?type=performers&q=${actorName}`, '.model');
if (searchRes.ok) {
const actorItem = searchRes.items.find(() => qu.query.exists(`.model-link img[alt="${actorName}"]`));
if (actorItem) {
const actorPath = actorItem.query.url('.model-link');
const actorUrl = `https://kink.com${actorPath}`;
const actorRes = await qu.get(actorUrl);
if (actorRes.ok) {
return scrapeProfile(actorRes.item, actorUrl, include);
}
return actorRes.status;
}
return null;
}
return searchRes.status;
}
module.exports = { module.exports = {
fetchLatest, fetchLatest,
fetchScene, fetchScene,
fetchProfile,
}; };

View File

@ -4,7 +4,7 @@ const qu = require('../utils/qu');
const slugify = require('../utils/slugify'); const slugify = require('../utils/slugify');
const { feetInchesToCm, lbsToKg } = require('../utils/convert'); const { feetInchesToCm, lbsToKg } = require('../utils/convert');
function scrapeAll(scenes) { function scrapeAll(scenes, entity) {
return scenes.map(({ query }) => { return scenes.map(({ query }) => {
const release = {}; const release = {};
@ -20,6 +20,7 @@ function scrapeAll(scenes) {
release.actors = query.all('.tour_update_models a', true); release.actors = query.all('.tour_update_models a', true);
release.poster = query.img('.videoPic img'); release.poster = query.img('.videoPic img');
release.entity = entity;
return release; return release;
}); });
@ -76,10 +77,21 @@ function scrapeProfile({ query }) {
} }
async function fetchLatest(channel, page = 1) { async function fetchLatest(channel, page = 1) {
const url = `https://pervcity.com/search.php?site[]=${channel.parameters.siteId}&page=${page}`; if (channel.parameters?.siteId) {
const res = await qu.getAll(url, '.videoBlock'); const url = `https://pervcity.com/search.php?site[]=${channel.parameters.siteId}&page=${page}`;
const res = await qu.getAll(url, '.videoBlock');
return res.ok ? scrapeAll(res.items, channel) : res.status; return res.ok ? scrapeAll(res.items, channel) : res.status;
}
return null;
}
async function fetchUpcoming(channel) {
const url = 'https://pervcity.com';
const res = await qu.getAll(url, '.upcoming .videoBlock');
return res.ok ? scrapeAll(res.items, channel.parent) : res.status;
} }
async function fetchScene(url, entity) { async function fetchScene(url, entity) {
@ -110,4 +122,5 @@ module.exports = {
fetchLatest, fetchLatest,
fetchScene, fetchScene,
fetchProfile, fetchProfile,
fetchUpcoming,
}; };

View File

@ -83,7 +83,7 @@ module.exports = {
brazzers, brazzers,
burningangel, burningangel,
cherrypimps, cherrypimps,
ddfnetwork, pornworld: ddfnetwork,
digitalplayground, digitalplayground,
dogfart, dogfart,
dogfartnetwork: dogfart, dogfartnetwork: dogfart,
@ -155,7 +155,7 @@ module.exports = {
brazzers, brazzers,
burningangel, burningangel,
cherrypimps, cherrypimps,
ddfnetwork, pornworld: ddfnetwork,
deeper: vixen, deeper: vixen,
deeplush: nubiles, deeplush: nubiles,
devilsfilm: famedigital, devilsfilm: famedigital,
@ -180,6 +180,7 @@ module.exports = {
julesjordan, julesjordan,
kellymadison, kellymadison,
killergram, killergram,
kink,
legalporno, legalporno,
men, men,
metrohd, metrohd,

View File

@ -184,10 +184,10 @@ async function updateReleasesSearch(releaseIds) {
COALESCE(releases.title, '') || ' ' || COALESCE(releases.title, '') || ' ' ||
entities.name || ' ' || entities.name || ' ' ||
entities.slug || ' ' || entities.slug || ' ' ||
COALESCE(entities.alias, '') || ' ' || COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
COALESCE(parents.name, '') || ' ' || COALESCE(parents.name, '') || ' ' ||
COALESCE(parents.slug, '') || ' ' || COALESCE(parents.slug, '') || ' ' ||
COALESCE(parents.alias, '') || ' ' || COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
COALESCE(releases.shoot_id, '') || ' ' || COALESCE(releases.shoot_id, '') || ' ' ||
COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMmonth mon DD FMDD'), '') || ' ' || COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMmonth mon DD FMDD'), '') || ' ' ||
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' || STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||

View File

@ -94,7 +94,10 @@ async function scrapeReleases(scraper, entity, preData, upcoming = false) {
return accReleases; return accReleases;
} }
const latestReleasesWithEntity = latestReleases.map(release => ({ ...release, entity })); // attach entity the release is assigned to when stored const latestReleasesWithEntity = latestReleases.map(release => ({
...release,
entity: release.entity || entity, // allow override
})); // attach entity the release is assigned to when stored
const uniqueReleases = argv.redownload const uniqueReleases = argv.redownload
? latestReleasesWithEntity ? latestReleasesWithEntity

View File

@ -127,16 +127,16 @@ function date(context, selector, format, match, attr = 'textContent') {
function image(context, selector = 'img', attr, origin, protocol = 'https') { function image(context, selector = 'img', attr, origin, protocol = 'https') {
const imageEl = (attr && q(context, selector, attr)) const imageEl = (attr && q(context, selector, attr))
|| q(context, selector, 'src') || q(context, selector, 'data-src')
|| q(context, selector, 'data-src'); || q(context, selector, 'src');
return prefixUrl(imageEl, origin, protocol); return prefixUrl(imageEl, origin, protocol);
} }
function images(context, selector = 'img', attr, origin, protocol = 'https') { function images(context, selector = 'img', attr, origin, protocol = 'https') {
const attribute = attr const attribute = attr
|| (q(context, selector, 'src') && 'src') || (q(context, selector, 'data-src') && 'data-src')
|| (q(context, selector, 'data-src') && 'data-src'); || (q(context, selector, 'src') && 'src');
const imageEls = all(context, selector, attribute); const imageEls = all(context, selector, attribute);
@ -358,6 +358,7 @@ module.exports = {
ctxa: initAll, ctxa: initAll,
geta: getAll, geta: getAll,
qu: quFuncs, qu: quFuncs,
query: quFuncs,
prefixUrl, prefixUrl,
...legacyFuncs, ...legacyFuncs,
}; };