Merge branch 'experimental'
|
@ -94,7 +94,7 @@
|
||||||
|
|
||||||
<a
|
<a
|
||||||
v-else
|
v-else
|
||||||
:href="`/channel/${release.entity.slug}`"
|
:href="`/${release.entity.type}/${release.entity.slug}`"
|
||||||
>
|
>
|
||||||
<img
|
<img
|
||||||
:src="`/img/logos/${release.entity.slug}/thumbs/network.png`"
|
:src="`/img/logos/${release.entity.slug}/thumbs/network.png`"
|
||||||
|
|
|
@ -79,7 +79,7 @@ export default {
|
||||||
box-shadow: 0 0 3px var(--darken);
|
box-shadow: 0 0 3px var(--darken);
|
||||||
|
|
||||||
.favicon {
|
.favicon {
|
||||||
height: 1rem;
|
width: 1rem;
|
||||||
margin: 0 .25rem 0 0;
|
margin: 0 .25rem 0 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -121,6 +121,9 @@ module.exports = {
|
||||||
],
|
],
|
||||||
'21sextury',
|
'21sextury',
|
||||||
'julesjordan',
|
'julesjordan',
|
||||||
|
'bang',
|
||||||
|
'pervcity',
|
||||||
|
'kink',
|
||||||
'peternorth',
|
'peternorth',
|
||||||
'naughtyamerica',
|
'naughtyamerica',
|
||||||
'cherrypimps',
|
'cherrypimps',
|
||||||
|
|
|
@ -174,7 +174,7 @@ exports.up = knex => Promise.resolve()
|
||||||
|
|
||||||
table.unique(['slug', 'type']);
|
table.unique(['slug', 'type']);
|
||||||
|
|
||||||
table.text('alias');
|
table.specificType('alias', 'text[]');
|
||||||
|
|
||||||
table.text('url');
|
table.text('url');
|
||||||
table.text('description');
|
table.text('description');
|
||||||
|
@ -820,6 +820,8 @@ exports.up = knex => Promise.resolve()
|
||||||
WHERE
|
WHERE
|
||||||
name ILIKE ('%' || search || '%') OR
|
name ILIKE ('%' || search || '%') OR
|
||||||
slug ILIKE ('%' || search || '%') OR
|
slug ILIKE ('%' || search || '%') OR
|
||||||
|
array_to_string(alias, '') ILIKE ('%' || search || '%') OR
|
||||||
|
replace(array_to_string(alias, ''), ' ', '') ILIKE ('%' || search || '%') OR
|
||||||
url ILIKE ('%' || search || '%')
|
url ILIKE ('%' || search || '%')
|
||||||
$$ LANGUAGE SQL STABLE;
|
$$ LANGUAGE SQL STABLE;
|
||||||
|
|
||||||
|
|
Before Width: | Height: | Size: 34 KiB |
Before Width: | Height: | Size: 1.0 KiB |
Before Width: | Height: | Size: 28 KiB |
Before Width: | Height: | Size: 1.2 KiB |
Before Width: | Height: | Size: 1.2 KiB |
Before Width: | Height: | Size: 2.5 KiB After Width: | Height: | Size: 2.5 KiB |
Before Width: | Height: | Size: 2.4 KiB After Width: | Height: | Size: 2.4 KiB |
Before Width: | Height: | Size: 3.6 KiB After Width: | Height: | Size: 3.6 KiB |
Before Width: | Height: | Size: 3.7 KiB After Width: | Height: | Size: 3.7 KiB |
Before Width: | Height: | Size: 3.8 KiB After Width: | Height: | Size: 3.8 KiB |
Before Width: | Height: | Size: 3.5 KiB After Width: | Height: | Size: 3.5 KiB |
After Width: | Height: | Size: 3.8 KiB |
Before Width: | Height: | Size: 2.4 KiB After Width: | Height: | Size: 2.4 KiB |
After Width: | Height: | Size: 166 KiB |
Before Width: | Height: | Size: 27 KiB After Width: | Height: | Size: 27 KiB |
Before Width: | Height: | Size: 17 KiB After Width: | Height: | Size: 17 KiB |
Before Width: | Height: | Size: 36 KiB After Width: | Height: | Size: 36 KiB |
Before Width: | Height: | Size: 3.7 KiB After Width: | Height: | Size: 3.7 KiB |
Before Width: | Height: | Size: 30 KiB After Width: | Height: | Size: 30 KiB |
Before Width: | Height: | Size: 15 KiB After Width: | Height: | Size: 15 KiB |
After Width: | Height: | Size: 30 KiB |
Before Width: | Height: | Size: 20 KiB After Width: | Height: | Size: 20 KiB |
Before Width: | Height: | Size: 10 KiB After Width: | Height: | Size: 10 KiB |
After Width: | Height: | Size: 36 KiB |
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 12 KiB |
Before Width: | Height: | Size: 16 KiB After Width: | Height: | Size: 16 KiB |
Before Width: | Height: | Size: 22 KiB After Width: | Height: | Size: 22 KiB |
Before Width: | Height: | Size: 18 KiB After Width: | Height: | Size: 19 KiB |
Before Width: | Height: | Size: 14 KiB After Width: | Height: | Size: 15 KiB |
After Width: | Height: | Size: 1.9 KiB |
After Width: | Height: | Size: 29 KiB |
Before Width: | Height: | Size: 19 KiB After Width: | Height: | Size: 19 KiB |
Before Width: | Height: | Size: 14 KiB After Width: | Height: | Size: 15 KiB |
Before Width: | Height: | Size: 18 KiB After Width: | Height: | Size: 18 KiB |
Before Width: | Height: | Size: 2.5 KiB After Width: | Height: | Size: 2.5 KiB |
Before Width: | Height: | Size: 2.6 KiB After Width: | Height: | Size: 2.6 KiB |
Before Width: | Height: | Size: 2.4 KiB After Width: | Height: | Size: 2.4 KiB |
Before Width: | Height: | Size: 2.3 KiB After Width: | Height: | Size: 2.3 KiB |
Before Width: | Height: | Size: 2.7 KiB After Width: | Height: | Size: 2.7 KiB |
Before Width: | Height: | Size: 2.4 KiB After Width: | Height: | Size: 2.4 KiB |
Before Width: | Height: | Size: 2.4 KiB After Width: | Height: | Size: 2.4 KiB |
After Width: | Height: | Size: 1.2 KiB |
Before Width: | Height: | Size: 2.3 KiB After Width: | Height: | Size: 2.3 KiB |
Before Width: | Height: | Size: 2.4 KiB After Width: | Height: | Size: 2.4 KiB |
Before Width: | Height: | Size: 2.4 KiB After Width: | Height: | Size: 2.4 KiB |
Before Width: | Height: | Size: 2.7 KiB After Width: | Height: | Size: 2.7 KiB |
After Width: | Height: | Size: 2.7 KiB |
Before Width: | Height: | Size: 2.7 KiB After Width: | Height: | Size: 2.7 KiB |
After Width: | Height: | Size: 1.7 KiB |
Before Width: | Height: | Size: 40 KiB After Width: | Height: | Size: 40 KiB |
Before Width: | Height: | Size: 7.8 KiB After Width: | Height: | Size: 7.8 KiB |
Before Width: | Height: | Size: 27 KiB After Width: | Height: | Size: 27 KiB |
Before Width: | Height: | Size: 2.0 KiB After Width: | Height: | Size: 2.0 KiB |
Before Width: | Height: | Size: 27 KiB After Width: | Height: | Size: 27 KiB |
Before Width: | Height: | Size: 44 KiB After Width: | Height: | Size: 44 KiB |
Before Width: | Height: | Size: 3.4 KiB After Width: | Height: | Size: 3.4 KiB |
After Width: | Height: | Size: 39 KiB |
Before Width: | Height: | Size: 18 KiB After Width: | Height: | Size: 18 KiB |
After Width: | Height: | Size: 27 KiB |
Before Width: | Height: | Size: 8.9 KiB After Width: | Height: | Size: 8.9 KiB |
Before Width: | Height: | Size: 8.9 KiB After Width: | Height: | Size: 8.9 KiB |
Before Width: | Height: | Size: 8.5 KiB After Width: | Height: | Size: 8.5 KiB |
Before Width: | Height: | Size: 8.1 KiB After Width: | Height: | Size: 8.1 KiB |
Before Width: | Height: | Size: 9.1 KiB After Width: | Height: | Size: 9.1 KiB |
Before Width: | Height: | Size: 7.7 KiB After Width: | Height: | Size: 7.7 KiB |
Before Width: | Height: | Size: 7.2 KiB After Width: | Height: | Size: 7.2 KiB |
After Width: | Height: | Size: 1.2 KiB |
Before Width: | Height: | Size: 8.5 KiB After Width: | Height: | Size: 8.5 KiB |
Before Width: | Height: | Size: 8.1 KiB After Width: | Height: | Size: 8.1 KiB |
Before Width: | Height: | Size: 6.9 KiB After Width: | Height: | Size: 6.9 KiB |
Before Width: | Height: | Size: 8.6 KiB After Width: | Height: | Size: 8.6 KiB |
After Width: | Height: | Size: 16 KiB |
Before Width: | Height: | Size: 8.6 KiB After Width: | Height: | Size: 8.6 KiB |
After Width: | Height: | Size: 16 KiB |
|
@ -136,9 +136,13 @@ const networks = [
|
||||||
type: 'info',
|
type: 'info',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
slug: 'ddfnetwork',
|
slug: 'pornworld',
|
||||||
name: 'DDF Network',
|
name: 'Porn World',
|
||||||
url: 'https://ddfnetwork.com',
|
alias: [
|
||||||
|
'ddf network',
|
||||||
|
'denys defrancesco',
|
||||||
|
],
|
||||||
|
url: 'https://pornworld.com',
|
||||||
description: 'European porn videos hub with exclusive VR, 4K and full HD XXX videos and hot sex photos of Europes finest porn star babes.',
|
description: 'European porn videos hub with exclusive VR, 4K and full HD XXX videos and hot sex photos of Europes finest porn star babes.',
|
||||||
parent: 'wgcz',
|
parent: 'wgcz',
|
||||||
},
|
},
|
||||||
|
@ -417,7 +421,7 @@ exports.seed = knex => Promise.resolve()
|
||||||
slug: network.slug,
|
slug: network.slug,
|
||||||
name: network.name,
|
name: network.name,
|
||||||
type: network.type || 'network',
|
type: network.type || 'network',
|
||||||
alias: network.alias ? network.alias.join(',') : null,
|
alias: network.alias,
|
||||||
url: network.url,
|
url: network.url,
|
||||||
description: network.description,
|
description: network.description,
|
||||||
parameters: network.parameters,
|
parameters: network.parameters,
|
||||||
|
|
|
@ -1455,7 +1455,7 @@ const sites = [
|
||||||
alias: ['ddfb'],
|
alias: ['ddfb'],
|
||||||
url: 'https://ddfbusty.com',
|
url: 'https://ddfbusty.com',
|
||||||
description: 'Gorgeous Babes with big tits and Euro pornstars with huge natural boobs filmed in Exclusive Full HD, 4K, & VR porn videos.',
|
description: 'Gorgeous Babes with big tits and Euro pornstars with huge natural boobs filmed in Exclusive Full HD, 4K, & VR porn videos.',
|
||||||
parent: 'ddfnetwork',
|
parent: 'pornworld',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
slug: 'handsonhardcore',
|
slug: 'handsonhardcore',
|
||||||
|
@ -1463,7 +1463,7 @@ const sites = [
|
||||||
alias: ['hoh'],
|
alias: ['hoh'],
|
||||||
url: 'https://handsonhardcore.com',
|
url: 'https://handsonhardcore.com',
|
||||||
description: 'Hardcore Sex & Anal Fucking Exclusive XXX Videos in VR, 4K and full HD with Hot European Pornstars',
|
description: 'Hardcore Sex & Anal Fucking Exclusive XXX Videos in VR, 4K and full HD with Hot European Pornstars',
|
||||||
parent: 'ddfnetwork',
|
parent: 'pornworld',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
slug: 'houseoftaboo',
|
slug: 'houseoftaboo',
|
||||||
|
@ -1471,7 +1471,7 @@ const sites = [
|
||||||
alias: ['hotb', 'hotab'],
|
alias: ['hotb', 'hotab'],
|
||||||
url: 'https://houseoftaboo.com',
|
url: 'https://houseoftaboo.com',
|
||||||
description: 'Exclusive BDSM Porn & Extreme Sex Videos Produced in VR, 4K and full HD with The Hottest European Fetish Pornstars',
|
description: 'Exclusive BDSM Porn & Extreme Sex Videos Produced in VR, 4K and full HD with The Hottest European Fetish Pornstars',
|
||||||
parent: 'ddfnetwork',
|
parent: 'pornworld',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
slug: 'ddfnetworkvr',
|
slug: 'ddfnetworkvr',
|
||||||
|
@ -1479,21 +1479,24 @@ const sites = [
|
||||||
alias: ['ddfvr'],
|
alias: ['ddfvr'],
|
||||||
url: 'https://ddfnetworkvr.com',
|
url: 'https://ddfnetworkvr.com',
|
||||||
description: 'VR Porn Videos shot Exclusively in 180 3D 4K Virtual Reality featuring the Hottest European & American VR Pornstar Babes',
|
description: 'VR Porn Videos shot Exclusively in 180 3D 4K Virtual Reality featuring the Hottest European & American VR Pornstar Babes',
|
||||||
parent: 'ddfnetwork',
|
parent: 'pornworld',
|
||||||
|
parameters: {
|
||||||
|
latest: 'https://ddfnetworkvr.com/home/tagfiltered/keywords/-/',
|
||||||
|
},
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
slug: 'eurogirlsongirls',
|
slug: 'eurogirlsongirls',
|
||||||
name: 'Euro Girls on Girls',
|
name: 'Euro Girls on Girls',
|
||||||
url: 'https://eurogirlsongirls.com',
|
url: 'https://eurogirlsongirls.com',
|
||||||
description: 'Hot Lesbian Sex & Glamour Lesbian Porn Videos and Photos Starring Gorgeous European Pornstars in 4K and Full HD VR.',
|
description: 'Hot Lesbian Sex & Glamour Lesbian Porn Videos and Photos Starring Gorgeous European Pornstars in 4K and Full HD VR.',
|
||||||
parent: 'ddfnetwork',
|
parent: 'pornworld',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
slug: '1byday',
|
slug: '1byday',
|
||||||
name: '1By-Day',
|
name: '1By-Day',
|
||||||
url: 'https://1by-day.com',
|
url: 'https://1by-day.com',
|
||||||
description: 'Ultra Sexy Exclusive Solo Masturbation Videos in VR, 4K and full HD showcasing Glamour Babes & Intense Orgasms',
|
description: 'Ultra Sexy Exclusive Solo Masturbation Videos in VR, 4K and full HD showcasing Glamour Babes & Intense Orgasms',
|
||||||
parent: 'ddfnetwork',
|
parent: 'pornworld',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
slug: 'euroteenerotica',
|
slug: 'euroteenerotica',
|
||||||
|
@ -1501,14 +1504,14 @@ const sites = [
|
||||||
alias: ['ete'],
|
alias: ['ete'],
|
||||||
url: 'https://euroteenerotica.com',
|
url: 'https://euroteenerotica.com',
|
||||||
description: 'Teen Threesomes & Barely Legal Porn Videos in 4K, VR and FULL HD with Hot Nymphomaniac Teen Babes',
|
description: 'Teen Threesomes & Barely Legal Porn Videos in 4K, VR and FULL HD with Hot Nymphomaniac Teen Babes',
|
||||||
parent: 'ddfnetwork',
|
parent: 'pornworld',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
slug: 'hotlegsandfeet',
|
slug: 'hotlegsandfeet',
|
||||||
name: 'Hot Legs and Feet',
|
name: 'Hot Legs and Feet',
|
||||||
url: 'https://hotlegsandfeet.com',
|
url: 'https://hotlegsandfeet.com',
|
||||||
description: 'Foot Fetish & Sexy Legs Porn Videos with Hot and Sexy Euro Pornstars',
|
description: 'Foot Fetish & Sexy Legs Porn Videos with Hot and Sexy Euro Pornstars',
|
||||||
parent: 'ddfnetwork',
|
parent: 'pornworld',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
slug: 'onlyblowjob',
|
slug: 'onlyblowjob',
|
||||||
|
@ -1516,22 +1519,24 @@ const sites = [
|
||||||
alias: ['obj'],
|
alias: ['obj'],
|
||||||
url: 'https://onlyblowjob.com',
|
url: 'https://onlyblowjob.com',
|
||||||
description: 'Fantasy Blowjobs & POV Cock Sucking Videos and Photos Produced in VR, 4K and full HD featuring Sexy European Pornstars',
|
description: 'Fantasy Blowjobs & POV Cock Sucking Videos and Photos Produced in VR, 4K and full HD featuring Sexy European Pornstars',
|
||||||
parent: 'ddfnetwork',
|
parent: 'pornworld',
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
slug: 'fuckinhd',
|
slug: 'fuckinhd',
|
||||||
name: 'Fuck in HD',
|
name: 'Fuck in HD',
|
||||||
url: 'https://fuckinhd.com',
|
url: 'https://fuckinhd.com',
|
||||||
description: 'HD Hardcore Sex & XXX Fantasy Porn Videos and Photos Produced in full HD featuring a Variety of Hardcore Porn Niches.',
|
description: 'HD Hardcore Sex & XXX Fantasy Porn Videos and Photos Produced in full HD featuring a Variety of Hardcore Porn Niches.',
|
||||||
parent: 'ddfnetwork',
|
parent: 'pornworld',
|
||||||
parameters: { native: true },
|
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
slug: 'bustylover',
|
slug: 'bustylover',
|
||||||
name: 'Busty Lover',
|
name: 'Busty Lover',
|
||||||
url: 'https://bustylover.com',
|
url: 'https://bustylover.com',
|
||||||
parent: 'ddfnetwork',
|
parent: 'pornworld',
|
||||||
parameters: { native: true },
|
parameters: {
|
||||||
|
latest: 'https://bustylover.com/videos/',
|
||||||
|
blockLayout: true,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
// DIGITAL PLAYGROUND
|
// DIGITAL PLAYGROUND
|
||||||
{
|
{
|
||||||
|
@ -7434,7 +7439,7 @@ exports.seed = knex => Promise.resolve()
|
||||||
slug: site.slug,
|
slug: site.slug,
|
||||||
name: site.name,
|
name: site.name,
|
||||||
type: site.type || 'channel',
|
type: site.type || 'channel',
|
||||||
alias: (site.alias || []).join(','),
|
alias: site.alias,
|
||||||
description: site.description,
|
description: site.description,
|
||||||
url: site.url,
|
url: site.url,
|
||||||
parameters: site.parameters,
|
parameters: site.parameters,
|
||||||
|
|
|
@ -64,9 +64,10 @@ const ethnicities = {
|
||||||
black: 'black',
|
black: 'black',
|
||||||
caucasian: 'white',
|
caucasian: 'white',
|
||||||
european: 'white',
|
european: 'white',
|
||||||
hispanic: 'latina',
|
hispanic: 'latin',
|
||||||
indian: 'indian',
|
indian: 'indian',
|
||||||
japanese: 'japanese',
|
japanese: 'japanese',
|
||||||
|
latin: 'latin',
|
||||||
latina: 'latina',
|
latina: 'latina',
|
||||||
latino: 'latino',
|
latino: 'latino',
|
||||||
white: 'white',
|
white: 'white',
|
||||||
|
|
11
src/media.js
|
@ -591,13 +591,20 @@ async function storeMedias(baseMedias) {
|
||||||
);
|
);
|
||||||
|
|
||||||
const [uniqueHashMedias, existingHashMedias] = await findHashDuplicates(fetchedMedias);
|
const [uniqueHashMedias, existingHashMedias] = await findHashDuplicates(fetchedMedias);
|
||||||
const newMedias = argv.force ? uniqueHashMedias.concat(existingHashMedias) : uniqueHashMedias;
|
|
||||||
|
|
||||||
const savedMedias = await Promise.map(
|
const savedMedias = await Promise.map(
|
||||||
newMedias,
|
uniqueHashMedias,
|
||||||
async baseMedia => storeFile(baseMedia),
|
async baseMedia => storeFile(baseMedia),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
if (argv.force) {
|
||||||
|
// overwrite files in case image processing was changed
|
||||||
|
await Promise.map(
|
||||||
|
existingHashMedias,
|
||||||
|
async baseMedia => storeFile(baseMedia),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const newMediaWithEntries = savedMedias.map((media, index) => curateMediaEntry(media, index));
|
const newMediaWithEntries = savedMedias.map((media, index) => curateMediaEntry(media, index));
|
||||||
const newMediaEntries = newMediaWithEntries.filter(media => media.newEntry).map(media => media.entry);
|
const newMediaEntries = newMediaWithEntries.filter(media => media.newEntry).map(media => media.entry);
|
||||||
|
|
||||||
|
|
|
@ -2,53 +2,60 @@
|
||||||
|
|
||||||
const bhttp = require('bhttp');
|
const bhttp = require('bhttp');
|
||||||
|
|
||||||
const { ed, ex, exa, get } = require('../utils/q');
|
const qu = require('../utils/qu');
|
||||||
const slugify = require('../utils/slugify');
|
const slugify = require('../utils/slugify');
|
||||||
|
|
||||||
/* eslint-disable newline-per-chained-call */
|
function scrapeAll(scenes, site, origin) {
|
||||||
function scrapeAll(html, site, origin) {
|
return scenes.map(({ query }) => {
|
||||||
return exa(html, '.card.m-1:not(.pornstar-card)').map(({ q, qa, qd }) => {
|
|
||||||
const release = {};
|
const release = {};
|
||||||
|
|
||||||
release.title = q('a', 'title');
|
release.title = query.q('.card-title a, .videoContent h4 a', 'title');
|
||||||
release.url = `${site?.url || origin || 'https://ddfnetwork.com'}${q('a', 'href')}`;
|
release.url = `${site?.url || origin || 'https://pornworld.com'}${query.q('a', 'href')}`;
|
||||||
[release.entryId] = release.url.split('/').slice(-1);
|
release.entryId = release.url.match(/\/(\d+)$/)[1];
|
||||||
|
|
||||||
release.date = qd('small[datetime]', 'YYYY-MM-DD HH:mm:ss', null, 'datetime');
|
release.date = query.date('small[datetime]', 'YYYY-MM-DD HH:mm:ss', null, 'datetime');
|
||||||
release.actors = qa('.card-subtitle a', true).filter(Boolean);
|
release.actors = query.all('.card-subtitle a, .featuring a', true).filter(Boolean);
|
||||||
|
|
||||||
const duration = parseInt(q('.card-info div:nth-child(2) .card-text', true), 10) * 60;
|
release.description = query.q('h4 + p', true);
|
||||||
|
|
||||||
|
const duration = parseInt(query.q('.card-info div:nth-child(2) .card-text', true), 10) * 60;
|
||||||
if (duration) release.duration = duration;
|
if (duration) release.duration = duration;
|
||||||
|
else release.duration = query.dur('.time');
|
||||||
|
|
||||||
release.poster = q('img').dataset.src;
|
release.poster = query.img();
|
||||||
|
|
||||||
return release;
|
return release;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async function scrapeScene(html, url, _site) {
|
async function scrapeScene({ query }, url, _site) {
|
||||||
const { qu } = ex(html);
|
|
||||||
const release = {};
|
const release = {};
|
||||||
|
|
||||||
[release.entryId] = url.split('/').slice(-1);
|
release.entryId = url.match(/\/(\d+)$/)[1];
|
||||||
|
|
||||||
release.title = qu.meta('itemprop=name');
|
release.title = query.meta('itemprop=name') || query.q('.video-title h1', true) || query.q('.about-text .story-title') || query.q('h3', true);
|
||||||
release.description = qu.q('.descr-box p', true);
|
release.description = query.q('.descr-box p', true) || query.q('.about-text p:not(.story-title)', true) || query.text('.description p');
|
||||||
release.date = qu.date('meta[itemprop=uploadDate]', 'YYYY-MM-DD', null, 'content')
|
|
||||||
|| qu.date('.title-border:nth-child(2) p', 'MM.DD.YYYY');
|
|
||||||
|
|
||||||
release.actors = qu.all('.pornstar-card > a', 'title');
|
release.date = query.date('meta[itemprop=uploadDate]', 'YYYY-MM-DD', null, 'content')
|
||||||
release.tags = qu.all('.tags-tab .tags a', true);
|
|| query.date('.actors time', 'MMMM DD, YYYY')
|
||||||
|
|| query.date('.title-border:nth-child(2) p', 'MM.DD.YYYY')
|
||||||
|
|| query.date('.length', 'MMMM DD, YYYY', /\w+ \d{2}, \d{4}/);
|
||||||
|
|
||||||
release.duration = parseInt(qu.q('.icon-video-red + span', true), 10) * 60;
|
if (query.exists('.pornstar-card > a')) release.actors = query.all('.pornstar-card > a', 'title');
|
||||||
release.likes = Number(qu.q('.icon-like-red + span', true));
|
else if (query.exists('.actors a')) release.actors = query.all('.actors a', true);
|
||||||
|
|
||||||
release.poster = qu.poster();
|
if (query.exists('.tags-tab')) release.tags = query.all('.tags-tab .tags a', true);
|
||||||
release.photos = qu.urls('.photo-slider-guest .card a');
|
else if (query.exists('.tags-box')) release.tags = query.all('.tags-box .tags li', true);
|
||||||
|
|
||||||
release.trailer = qu.all('source[type="video/mp4"]').map(trailer => ({
|
release.duration = parseInt(query.q('.icon-video-red + span', true), 10) * 60 || query.dur('.length') || null;
|
||||||
|
release.likes = Number(query.q('.icon-like-red + span', true)) || null;
|
||||||
|
|
||||||
|
release.poster = query.poster() || query.poster('dl8-video');
|
||||||
|
release.photos = query.urls('.photo-slider-guest .card a');
|
||||||
|
|
||||||
|
release.trailer = query.all('source[type="video/mp4"]').map(trailer => ({
|
||||||
src: trailer.src,
|
src: trailer.src,
|
||||||
quality: Number(trailer.attributes.res.value),
|
quality: Number(trailer.attributes.res?.value || trailer.attributes.quality?.value.slice(0, -1)) || null,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
return release;
|
return release;
|
||||||
|
@ -59,9 +66,9 @@ async function fetchActorReleases(urls) {
|
||||||
const sources = urls.filter(url => !/ddfnetwork/.test(url));
|
const sources = urls.filter(url => !/ddfnetwork/.test(url));
|
||||||
|
|
||||||
const releases = await Promise.all(sources.map(async (url) => {
|
const releases = await Promise.all(sources.map(async (url) => {
|
||||||
const { html } = await get(url);
|
const res = await qu.getAll(url, '.card.m-1:not(.pornstar-card)');
|
||||||
|
|
||||||
return scrapeAll(html, null, new URL(url).origin);
|
return res.ok ? scrapeAll(res.items, null, new URL(url).origin) : null;
|
||||||
}));
|
}));
|
||||||
|
|
||||||
// DDF cross-releases scenes between sites, filter duplicates by entryId
|
// DDF cross-releases scenes between sites, filter duplicates by entryId
|
||||||
|
@ -71,11 +78,9 @@ async function fetchActorReleases(urls) {
|
||||||
.reduce((acc, release) => ({ ...acc, [release.entryId]: release }), {}));
|
.reduce((acc, release) => ({ ...acc, [release.entryId]: release }), {}));
|
||||||
}
|
}
|
||||||
|
|
||||||
async function scrapeProfile(html, _url, actorName) {
|
async function scrapeProfile({ query }, _url, actorName) {
|
||||||
const { qu } = ex(html);
|
const keys = query.all('.about-title', true).map(key => slugify(key, '_'));
|
||||||
|
const values = query.all('.about-info').map((el) => {
|
||||||
const keys = qu.all('.about-title', true).map(key => slugify(key, '_'));
|
|
||||||
const values = qu.all('.about-info').map((el) => {
|
|
||||||
if (el.children.length > 0) {
|
if (el.children.length > 0) {
|
||||||
return Array.from(el.children, child => child.textContent.trim()).join(', ');
|
return Array.from(el.children, child => child.textContent.trim()).join(', ');
|
||||||
}
|
}
|
||||||
|
@ -96,8 +101,8 @@ async function scrapeProfile(html, _url, actorName) {
|
||||||
name: actorName,
|
name: actorName,
|
||||||
};
|
};
|
||||||
|
|
||||||
profile.description = qu.q('.description-box', true);
|
profile.description = query.q('.description-box', true);
|
||||||
profile.birthdate = ed(bio.birthday, 'MMMM DD, YYYY');
|
profile.birthdate = qu.extractDate(bio.birthday, 'MMMM DD, YYYY');
|
||||||
|
|
||||||
if (bio.nationality) profile.nationality = bio.nationality;
|
if (bio.nationality) profile.nationality = bio.nationality;
|
||||||
|
|
||||||
|
@ -118,34 +123,37 @@ async function scrapeProfile(html, _url, actorName) {
|
||||||
|
|
||||||
if (bio.shoe_size) profile.shoes = Number(bio.shoe_size.split('|')[1]);
|
if (bio.shoe_size) profile.shoes = Number(bio.shoe_size.split('|')[1]);
|
||||||
|
|
||||||
const avatarEl = qu.q('.pornstar-details .card-img-top');
|
const avatarEl = query.q('.pornstar-details .card-img-top');
|
||||||
if (avatarEl && avatarEl.dataset.src.match('^//')) profile.avatar = `https:${avatarEl.dataset.src}`;
|
if (avatarEl && avatarEl.dataset.src.match('^//')) profile.avatar = `https:${avatarEl.dataset.src}`;
|
||||||
|
|
||||||
profile.releases = await fetchActorReleases(qu.urls('.find-me-tab li a'));
|
profile.releases = await fetchActorReleases(query.urls('.find-me-tab li a'));
|
||||||
|
|
||||||
return profile;
|
return profile;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchLatest(site, page = 1) {
|
async function fetchLatest(channel, page = 1) {
|
||||||
|
/* ddfnetwork.com redirects to pornworld.com
|
||||||
const url = site.parameters?.native
|
const url = site.parameters?.native
|
||||||
? `${site.url}/videos/search/latest/ever/allsite/-/${page}`
|
? `${site.url}/videos/search/latest/ever/allsite/-/${page}`
|
||||||
: `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`;
|
: `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`;
|
||||||
|
*/
|
||||||
|
|
||||||
const res = await bhttp.get(url);
|
const url = channel.parameters?.latest || `${channel.url}/videos/search/latest/ever/allsite/-/${page}`;
|
||||||
|
const res = await qu.getAll(url, '.card.m-1:not(.pornstar-card), .allVideos .videoBlock');
|
||||||
|
|
||||||
if (res.statusCode === 200) {
|
if (res.ok) {
|
||||||
return scrapeAll(res.body.toString(), site);
|
return scrapeAll(res.items, channel);
|
||||||
}
|
}
|
||||||
|
|
||||||
return res.statusCode;
|
return res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchScene(url, site) {
|
async function fetchScene(url, site) {
|
||||||
// DDF's main site moved to Porn World
|
// DDF's main site moved to Porn World
|
||||||
// const res = await bhttp.get(`https://ddfnetwork.com${new URL(url).pathname}`);
|
// const res = await bhttp.get(`https://ddfnetwork.com${new URL(url).pathname}`);
|
||||||
const res = await bhttp.get(url);
|
const res = await qu.get(url, '.content, #content, .taspVideoPage');
|
||||||
|
|
||||||
return scrapeScene(res.body.toString(), url, site);
|
return res.ok ? scrapeScene(res.item, url, site) : res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchProfile(actorName) {
|
async function fetchProfile(actorName) {
|
||||||
|
@ -155,7 +163,7 @@ async function fetchProfile(actorName) {
|
||||||
word: actorName,
|
word: actorName,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
decodeJSON: true,
|
decodeJSON: false,
|
||||||
headers: {
|
headers: {
|
||||||
'x-requested-with': 'XMLHttpRequest',
|
'x-requested-with': 'XMLHttpRequest',
|
||||||
},
|
},
|
||||||
|
|
|
@ -1,54 +1,54 @@
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const { get, getAll } = require('../utils/qu');
|
const qu = require('../utils/qu');
|
||||||
|
|
||||||
function scrapeLatest(scenes) {
|
function scrapeAll(scenes) {
|
||||||
return scenes.map(({ qu }) => {
|
return scenes.map(({ query }) => {
|
||||||
const release = {};
|
const release = {};
|
||||||
|
|
||||||
const href = qu.url('.shoot-thumb-title a');
|
const href = query.url('.shoot-thumb-title a');
|
||||||
release.url = `https://kink.com${href}`;
|
release.url = `https://kink.com${href}`;
|
||||||
|
|
||||||
release.shootId = href.split('/').slice(-1)[0];
|
release.shootId = href.split('/').slice(-1)[0];
|
||||||
release.entryId = release.shootId;
|
release.entryId = release.shootId;
|
||||||
|
|
||||||
release.title = qu.q('.shoot-thumb-title a', true);
|
release.title = query.q('.shoot-thumb-title a', true);
|
||||||
release.date = qu.date('.date', 'MMM DD, YYYY');
|
release.date = query.date('.date', 'MMM DD, YYYY');
|
||||||
|
|
||||||
release.actors = qu.all('.shoot-thumb-models a', true);
|
release.actors = query.all('.shoot-thumb-models a', true);
|
||||||
release.stars = qu.q('.average-rating', 'data-rating') / 10;
|
release.stars = query.q('.average-rating', 'data-rating') / 10;
|
||||||
|
|
||||||
release.poster = qu.img('.adimage');
|
release.poster = query.img('.adimage');
|
||||||
release.photos = qu.imgs('.rollover .roll-image', 'data-imagesrc').map(photo => [
|
release.photos = query.imgs('.rollover .roll-image', 'data-imagesrc').map(photo => [
|
||||||
photo.replace('410/', '830/'),
|
photo.replace('410/', '830/'),
|
||||||
photo,
|
photo,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
release.duration = qu.dur('.video span');
|
release.duration = query.dur('.video span');
|
||||||
|
|
||||||
return release;
|
return release;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async function scrapeScene({ qu }, url) {
|
async function scrapeScene({ query }, url) {
|
||||||
const release = { url };
|
const release = { url };
|
||||||
|
|
||||||
release.shootId = new URL(url).pathname.split('/')[2];
|
release.shootId = new URL(url).pathname.split('/')[2];
|
||||||
release.entryId = release.shootId;
|
release.entryId = release.shootId;
|
||||||
|
|
||||||
release.title = qu.q('.shoot-title span.favorite-button', 'data-title');
|
release.title = query.q('.shoot-title span.favorite-button', 'data-title');
|
||||||
release.description = qu.q('.description-text', true);
|
release.description = query.q('.description-text', true);
|
||||||
|
|
||||||
release.date = qu.date('.shoot-date', 'MMMM DD, YYYY');
|
release.date = query.date('.shoot-date', 'MMMM DD, YYYY');
|
||||||
release.actors = qu.all('.names a', true).map(actor => actor.replace(/,\s*/, ''));
|
release.actors = query.all('.names a', true).map(actor => actor.replace(/,\s*/, ''));
|
||||||
release.director = qu.q('.director-name', true);
|
release.director = query.q('.director-name', true);
|
||||||
|
|
||||||
release.photos = qu.imgs('.gallery .thumb img', 'data-image-file');
|
release.photos = query.imgs('.gallery .thumb img', 'data-image-file');
|
||||||
release.poster = qu.poster();
|
release.poster = query.poster();
|
||||||
|
|
||||||
release.tags = qu.all('.tag-list a[href*="/tag"]', true).map(tag => tag.replace(/,\s*/, ''));
|
release.tags = query.all('.tag-list a[href*="/tag"]', true).map(tag => tag.replace(/,\s*/, ''));
|
||||||
|
|
||||||
const trailer = qu.q('.player span[data-type="trailer-src"]', 'data-url');
|
const trailer = query.q('.player span[data-type="trailer-src"]', 'data-url');
|
||||||
|
|
||||||
release.trailer = [
|
release.trailer = [
|
||||||
{
|
{
|
||||||
|
@ -69,23 +69,77 @@ async function scrapeScene({ qu }, url) {
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
release.channel = qu.url('.shoot-logo a').split('/').slice(-1)[0];
|
release.channel = query.url('.shoot-logo a').split('/').slice(-1)[0];
|
||||||
|
|
||||||
return release;
|
return release;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchLatest(site, page = 1) {
|
async function fetchActorReleases(actorUrl, page = 1, accReleases = []) {
|
||||||
const res = await getAll(`${site.url}/latest/page/${page}`, '.shoot-list .shoot');
|
const res = await qu.get(`${actorUrl}?page=${page}`);
|
||||||
|
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
return scrapeLatest(res.items, site);
|
const releases = scrapeAll(qu.initAll(res.item.el, '.shoot-list .shoot'));
|
||||||
|
const hasNextPage = res.item.query.exists('.paginated-nav li:last-child:not(.disabled)');
|
||||||
|
|
||||||
|
if (hasNextPage) {
|
||||||
|
return fetchActorReleases(actorUrl, page + 1, accReleases.concat(releases));
|
||||||
|
}
|
||||||
|
|
||||||
|
return accReleases.concat(releases);
|
||||||
|
}
|
||||||
|
|
||||||
|
return accReleases;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function scrapeProfile({ query }, actorUrl, include) {
|
||||||
|
const profile = {};
|
||||||
|
|
||||||
|
profile.description = query.q('.bio #expand-text', true);
|
||||||
|
|
||||||
|
const tags = query.all('.bio-tags a', true);
|
||||||
|
|
||||||
|
if (tags.includes('brunette') || tags.includes('brunet')) profile.hairColor = 'brown';
|
||||||
|
if (tags.includes('blonde') || tags.includes('blond')) profile.hairColor = 'blonde';
|
||||||
|
if (tags.includes('black hair')) profile.hairColor = 'black';
|
||||||
|
if (tags.includes('redhead')) profile.hairColor = 'red';
|
||||||
|
|
||||||
|
if (tags.includes('natural boobs')) profile.naturalBoobs = true;
|
||||||
|
if (tags.includes('fake boobs')) profile.naturalBoobs = false;
|
||||||
|
|
||||||
|
if (tags.includes('white')) profile.ethnicity = 'white';
|
||||||
|
if (tags.includes('latin')) profile.ethnicity = 'latin';
|
||||||
|
if (tags.includes('Black')) profile.ethnicity = 'black';
|
||||||
|
|
||||||
|
if (tags.includes('pierced nipples')) profile.hasPiercings = true;
|
||||||
|
if (tags.includes('tattoo')) profile.hasTattoos = true;
|
||||||
|
|
||||||
|
if (tags.includes('foreskin')) profile.hasForeskin = true;
|
||||||
|
|
||||||
|
if ((tags.includes('big dick') || tags.includes('foreskin'))
|
||||||
|
&& (tags.includes('fake boobs') || tags.includes('big tits'))) profile.gender = 'transsexual';
|
||||||
|
|
||||||
|
profile.avatar = query.img('.bio-slider-img, .bio-img:not([src*="Missing"])');
|
||||||
|
profile.social = query.urls('a.social-link');
|
||||||
|
|
||||||
|
if (include.releases) {
|
||||||
|
profile.releases = await fetchActorReleases(actorUrl);
|
||||||
|
}
|
||||||
|
|
||||||
|
return profile;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchLatest(site, page = 1) {
|
||||||
|
const res = await qu.getAll(`${site.url}/latest/page/${page}`, '.shoot-list .shoot');
|
||||||
|
|
||||||
|
if (res.ok) {
|
||||||
|
return scrapeAll(res.items, site);
|
||||||
}
|
}
|
||||||
|
|
||||||
return res.status;
|
return res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchScene(url, site) {
|
async function fetchScene(url, site) {
|
||||||
const res = await get(url);
|
const res = await qu.get(url);
|
||||||
|
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
return scrapeScene(res.item, url, site);
|
return scrapeScene(res.item, url, site);
|
||||||
|
@ -94,7 +148,32 @@ async function fetchScene(url, site) {
|
||||||
return res.status;
|
return res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function fetchProfile(actorName, entity, include) {
|
||||||
|
const searchRes = await qu.getAll(`https://kink.com/search?type=performers&q=${actorName}`, '.model');
|
||||||
|
|
||||||
|
if (searchRes.ok) {
|
||||||
|
const actorItem = searchRes.items.find(() => qu.query.exists(`.model-link img[alt="${actorName}"]`));
|
||||||
|
|
||||||
|
if (actorItem) {
|
||||||
|
const actorPath = actorItem.query.url('.model-link');
|
||||||
|
const actorUrl = `https://kink.com${actorPath}`;
|
||||||
|
const actorRes = await qu.get(actorUrl);
|
||||||
|
|
||||||
|
if (actorRes.ok) {
|
||||||
|
return scrapeProfile(actorRes.item, actorUrl, include);
|
||||||
|
}
|
||||||
|
|
||||||
|
return actorRes.status;
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return searchRes.status;
|
||||||
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
fetchLatest,
|
fetchLatest,
|
||||||
fetchScene,
|
fetchScene,
|
||||||
|
fetchProfile,
|
||||||
};
|
};
|
||||||
|
|
|
@ -4,7 +4,7 @@ const qu = require('../utils/qu');
|
||||||
const slugify = require('../utils/slugify');
|
const slugify = require('../utils/slugify');
|
||||||
const { feetInchesToCm, lbsToKg } = require('../utils/convert');
|
const { feetInchesToCm, lbsToKg } = require('../utils/convert');
|
||||||
|
|
||||||
function scrapeAll(scenes) {
|
function scrapeAll(scenes, entity) {
|
||||||
return scenes.map(({ query }) => {
|
return scenes.map(({ query }) => {
|
||||||
const release = {};
|
const release = {};
|
||||||
|
|
||||||
|
@ -20,6 +20,7 @@ function scrapeAll(scenes) {
|
||||||
release.actors = query.all('.tour_update_models a', true);
|
release.actors = query.all('.tour_update_models a', true);
|
||||||
|
|
||||||
release.poster = query.img('.videoPic img');
|
release.poster = query.img('.videoPic img');
|
||||||
|
release.entity = entity;
|
||||||
|
|
||||||
return release;
|
return release;
|
||||||
});
|
});
|
||||||
|
@ -76,10 +77,21 @@ function scrapeProfile({ query }) {
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchLatest(channel, page = 1) {
|
async function fetchLatest(channel, page = 1) {
|
||||||
|
if (channel.parameters?.siteId) {
|
||||||
const url = `https://pervcity.com/search.php?site[]=${channel.parameters.siteId}&page=${page}`;
|
const url = `https://pervcity.com/search.php?site[]=${channel.parameters.siteId}&page=${page}`;
|
||||||
const res = await qu.getAll(url, '.videoBlock');
|
const res = await qu.getAll(url, '.videoBlock');
|
||||||
|
|
||||||
return res.ok ? scrapeAll(res.items, channel) : res.status;
|
return res.ok ? scrapeAll(res.items, channel) : res.status;
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchUpcoming(channel) {
|
||||||
|
const url = 'https://pervcity.com';
|
||||||
|
const res = await qu.getAll(url, '.upcoming .videoBlock');
|
||||||
|
|
||||||
|
return res.ok ? scrapeAll(res.items, channel.parent) : res.status;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchScene(url, entity) {
|
async function fetchScene(url, entity) {
|
||||||
|
@ -110,4 +122,5 @@ module.exports = {
|
||||||
fetchLatest,
|
fetchLatest,
|
||||||
fetchScene,
|
fetchScene,
|
||||||
fetchProfile,
|
fetchProfile,
|
||||||
|
fetchUpcoming,
|
||||||
};
|
};
|
||||||
|
|
|
@ -83,7 +83,7 @@ module.exports = {
|
||||||
brazzers,
|
brazzers,
|
||||||
burningangel,
|
burningangel,
|
||||||
cherrypimps,
|
cherrypimps,
|
||||||
ddfnetwork,
|
pornworld: ddfnetwork,
|
||||||
digitalplayground,
|
digitalplayground,
|
||||||
dogfart,
|
dogfart,
|
||||||
dogfartnetwork: dogfart,
|
dogfartnetwork: dogfart,
|
||||||
|
@ -155,7 +155,7 @@ module.exports = {
|
||||||
brazzers,
|
brazzers,
|
||||||
burningangel,
|
burningangel,
|
||||||
cherrypimps,
|
cherrypimps,
|
||||||
ddfnetwork,
|
pornworld: ddfnetwork,
|
||||||
deeper: vixen,
|
deeper: vixen,
|
||||||
deeplush: nubiles,
|
deeplush: nubiles,
|
||||||
devilsfilm: famedigital,
|
devilsfilm: famedigital,
|
||||||
|
@ -180,6 +180,7 @@ module.exports = {
|
||||||
julesjordan,
|
julesjordan,
|
||||||
kellymadison,
|
kellymadison,
|
||||||
killergram,
|
killergram,
|
||||||
|
kink,
|
||||||
legalporno,
|
legalporno,
|
||||||
men,
|
men,
|
||||||
metrohd,
|
metrohd,
|
||||||
|
|
|
@ -184,10 +184,10 @@ async function updateReleasesSearch(releaseIds) {
|
||||||
COALESCE(releases.title, '') || ' ' ||
|
COALESCE(releases.title, '') || ' ' ||
|
||||||
entities.name || ' ' ||
|
entities.name || ' ' ||
|
||||||
entities.slug || ' ' ||
|
entities.slug || ' ' ||
|
||||||
COALESCE(entities.alias, '') || ' ' ||
|
COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
|
||||||
COALESCE(parents.name, '') || ' ' ||
|
COALESCE(parents.name, '') || ' ' ||
|
||||||
COALESCE(parents.slug, '') || ' ' ||
|
COALESCE(parents.slug, '') || ' ' ||
|
||||||
COALESCE(parents.alias, '') || ' ' ||
|
COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
|
||||||
COALESCE(releases.shoot_id, '') || ' ' ||
|
COALESCE(releases.shoot_id, '') || ' ' ||
|
||||||
COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMmonth mon DD FMDD'), '') || ' ' ||
|
COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMmonth mon DD FMDD'), '') || ' ' ||
|
||||||
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
|
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
|
||||||
|
|
|
@ -94,7 +94,10 @@ async function scrapeReleases(scraper, entity, preData, upcoming = false) {
|
||||||
return accReleases;
|
return accReleases;
|
||||||
}
|
}
|
||||||
|
|
||||||
const latestReleasesWithEntity = latestReleases.map(release => ({ ...release, entity })); // attach entity the release is assigned to when stored
|
const latestReleasesWithEntity = latestReleases.map(release => ({
|
||||||
|
...release,
|
||||||
|
entity: release.entity || entity, // allow override
|
||||||
|
})); // attach entity the release is assigned to when stored
|
||||||
|
|
||||||
const uniqueReleases = argv.redownload
|
const uniqueReleases = argv.redownload
|
||||||
? latestReleasesWithEntity
|
? latestReleasesWithEntity
|
||||||
|
|
|
@ -127,16 +127,16 @@ function date(context, selector, format, match, attr = 'textContent') {
|
||||||
|
|
||||||
function image(context, selector = 'img', attr, origin, protocol = 'https') {
|
function image(context, selector = 'img', attr, origin, protocol = 'https') {
|
||||||
const imageEl = (attr && q(context, selector, attr))
|
const imageEl = (attr && q(context, selector, attr))
|
||||||
|| q(context, selector, 'src')
|
|| q(context, selector, 'data-src')
|
||||||
|| q(context, selector, 'data-src');
|
|| q(context, selector, 'src');
|
||||||
|
|
||||||
return prefixUrl(imageEl, origin, protocol);
|
return prefixUrl(imageEl, origin, protocol);
|
||||||
}
|
}
|
||||||
|
|
||||||
function images(context, selector = 'img', attr, origin, protocol = 'https') {
|
function images(context, selector = 'img', attr, origin, protocol = 'https') {
|
||||||
const attribute = attr
|
const attribute = attr
|
||||||
|| (q(context, selector, 'src') && 'src')
|
|| (q(context, selector, 'data-src') && 'data-src')
|
||||||
|| (q(context, selector, 'data-src') && 'data-src');
|
|| (q(context, selector, 'src') && 'src');
|
||||||
|
|
||||||
const imageEls = all(context, selector, attribute);
|
const imageEls = all(context, selector, attribute);
|
||||||
|
|
||||||
|
@ -358,6 +358,7 @@ module.exports = {
|
||||||
ctxa: initAll,
|
ctxa: initAll,
|
||||||
geta: getAll,
|
geta: getAll,
|
||||||
qu: quFuncs,
|
qu: quFuncs,
|
||||||
|
query: quFuncs,
|
||||||
prefixUrl,
|
prefixUrl,
|
||||||
...legacyFuncs,
|
...legacyFuncs,
|
||||||
};
|
};
|
||||||
|
|