Added teaser support. Added Score network with scraper for Scoreland. Improved q. Added assets.
|
@ -120,9 +120,7 @@ async function fetchNetwork() {
|
||||||
}
|
}
|
||||||
|
|
||||||
this.sites = this.network.sites
|
this.sites = this.network.sites
|
||||||
.filter(site => !site.independent)
|
.filter(site => !site.independent);
|
||||||
// .concat(this.studios)
|
|
||||||
.sort(({ name: nameA }, { name: nameB }) => nameA.localeCompare(nameB));
|
|
||||||
|
|
||||||
this.releases = this.network.releases;
|
this.releases = this.network.releases;
|
||||||
}
|
}
|
||||||
|
|
|
@ -27,6 +27,22 @@
|
||||||
class="item trailer-video"
|
class="item trailer-video"
|
||||||
controls
|
controls
|
||||||
>Sorry, the tailer cannot be played in your browser</video>
|
>Sorry, the tailer cannot be played in your browser</video>
|
||||||
|
|
||||||
|
<video
|
||||||
|
v-else-if="release.teaser && /^video\//.test(release.teaser.mime)"
|
||||||
|
:src="`/media/${release.teaser.path}`"
|
||||||
|
:poster="`/media/${(release.poster && release.poster.thumbnail)}`"
|
||||||
|
:alt="release.title"
|
||||||
|
class="item trailer-video"
|
||||||
|
controls
|
||||||
|
>Sorry, the tailer cannot be played in your browser</video>
|
||||||
|
|
||||||
|
<img
|
||||||
|
v-else-if="release.teaser && /^image\//.test(release.teaser.mime)"
|
||||||
|
:src="`/media/${release.teaser.path}`"
|
||||||
|
:alt="release.title"
|
||||||
|
class="item trailer-video"
|
||||||
|
>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<a
|
<a
|
||||||
|
|
|
@ -28,6 +28,7 @@ function curateRelease(release) {
|
||||||
|
|
||||||
if (release.photos) curatedRelease.photos = release.photos.map(({ media }) => media);
|
if (release.photos) curatedRelease.photos = release.photos.map(({ media }) => media);
|
||||||
if (release.trailer) curatedRelease.trailer = release.trailer.media;
|
if (release.trailer) curatedRelease.trailer = release.trailer.media;
|
||||||
|
if (release.teaser) curatedRelease.teaser = release.teaser.media;
|
||||||
if (release.actors) curatedRelease.actors = release.actors.map(({ actor }) => curateActor(actor, curatedRelease));
|
if (release.actors) curatedRelease.actors = release.actors.map(({ actor }) => curateActor(actor, curatedRelease));
|
||||||
|
|
||||||
return curatedRelease;
|
return curatedRelease;
|
||||||
|
|
|
@ -92,6 +92,18 @@ const releaseTrailerFragment = `
|
||||||
index
|
index
|
||||||
path
|
path
|
||||||
thumbnail
|
thumbnail
|
||||||
|
mime
|
||||||
|
}
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
||||||
|
const releaseTeaserFragment = `
|
||||||
|
teaser: releasesTeaserByReleaseId {
|
||||||
|
media {
|
||||||
|
index
|
||||||
|
path
|
||||||
|
thumbnail
|
||||||
|
mime
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
`;
|
`;
|
||||||
|
@ -153,6 +165,7 @@ const releaseFragment = `
|
||||||
${releasePosterFragment}
|
${releasePosterFragment}
|
||||||
${releasePhotosFragment}
|
${releasePhotosFragment}
|
||||||
${releaseTrailerFragment}
|
${releaseTrailerFragment}
|
||||||
|
${releaseTeaserFragment}
|
||||||
${siteFragment}
|
${siteFragment}
|
||||||
studio {
|
studio {
|
||||||
id
|
id
|
||||||
|
|
|
@ -18,12 +18,15 @@ function initNetworksActions(store, _router) {
|
||||||
name
|
name
|
||||||
slug
|
slug
|
||||||
url
|
url
|
||||||
sites {
|
sites(
|
||||||
|
orderBy: PRIORITY_DESC
|
||||||
|
) {
|
||||||
id
|
id
|
||||||
name
|
name
|
||||||
slug
|
slug
|
||||||
url
|
url
|
||||||
independent
|
independent
|
||||||
|
priority
|
||||||
network {
|
network {
|
||||||
id
|
id
|
||||||
name
|
name
|
||||||
|
|
|
@ -40,6 +40,7 @@ module.exports = {
|
||||||
'kellymadison',
|
'kellymadison',
|
||||||
'bangbros',
|
'bangbros',
|
||||||
'ddfnetwork',
|
'ddfnetwork',
|
||||||
|
'score',
|
||||||
'boobpedia',
|
'boobpedia',
|
||||||
'pornhub',
|
'pornhub',
|
||||||
'freeones',
|
'freeones',
|
||||||
|
|
|
@ -407,6 +407,19 @@ exports.up = knex => Promise.resolve()
|
||||||
|
|
||||||
table.unique('release_id');
|
table.unique('release_id');
|
||||||
}))
|
}))
|
||||||
|
.then(() => knex.schema.createTable('releases_teasers', (table) => {
|
||||||
|
table.integer('release_id', 16)
|
||||||
|
.notNullable()
|
||||||
|
.references('id')
|
||||||
|
.inTable('releases');
|
||||||
|
|
||||||
|
table.integer('media_id', 16)
|
||||||
|
.notNullable()
|
||||||
|
.references('id')
|
||||||
|
.inTable('media');
|
||||||
|
|
||||||
|
table.unique('release_id');
|
||||||
|
}))
|
||||||
.then(() => knex.schema.createTable('releases_photos', (table) => {
|
.then(() => knex.schema.createTable('releases_photos', (table) => {
|
||||||
table.integer('release_id', 16)
|
table.integer('release_id', 16)
|
||||||
.notNullable()
|
.notNullable()
|
||||||
|
@ -475,6 +488,7 @@ exports.down = knex => knex.raw(`
|
||||||
DROP TABLE IF EXISTS releases_photos CASCADE;
|
DROP TABLE IF EXISTS releases_photos CASCADE;
|
||||||
DROP TABLE IF EXISTS releases_covers CASCADE;
|
DROP TABLE IF EXISTS releases_covers CASCADE;
|
||||||
DROP TABLE IF EXISTS releases_trailers CASCADE;
|
DROP TABLE IF EXISTS releases_trailers CASCADE;
|
||||||
|
DROP TABLE IF EXISTS releases_teasers CASCADE;
|
||||||
DROP TABLE IF EXISTS releases_tags CASCADE;
|
DROP TABLE IF EXISTS releases_tags CASCADE;
|
||||||
DROP TABLE IF EXISTS actors_avatars CASCADE;
|
DROP TABLE IF EXISTS actors_avatars CASCADE;
|
||||||
DROP TABLE IF EXISTS actors_photos CASCADE;
|
DROP TABLE IF EXISTS actors_photos CASCADE;
|
||||||
|
|
|
@ -0,0 +1,10 @@
|
||||||
|
exports.up = async knex => Promise.resolve()
|
||||||
|
.then(() => knex.schema.table('sites', (table) => {
|
||||||
|
table.integer('priority', 3)
|
||||||
|
.defaultTo(0);
|
||||||
|
}));
|
||||||
|
|
||||||
|
exports.down = async knex => Promise.resolve()
|
||||||
|
.then(() => knex.schema.table('sites', (table) => {
|
||||||
|
table.dropColumn('priority');
|
||||||
|
}));
|
|
@ -20,7 +20,7 @@
|
||||||
"rollback": "knex-migrate down",
|
"rollback": "knex-migrate down",
|
||||||
"seed-make": "knex seed:make",
|
"seed-make": "knex seed:make",
|
||||||
"seed": "knex seed:run",
|
"seed": "knex seed:run",
|
||||||
"flush": "cli-confirm \"This completely purges the database, are you sure?\" && knex-migrate down && knex-migrate up && knex seed:run"
|
"flush": "cli-confirm \"This completely purges the database, are you sure?\" && knex-migrate down --to 0 && knex-migrate up && knex seed:run"
|
||||||
},
|
},
|
||||||
"repository": {
|
"repository": {
|
||||||
"type": "git",
|
"type": "git",
|
||||||
|
|
After Width: | Height: | Size: 21 KiB |
After Width: | Height: | Size: 9.0 KiB |
After Width: | Height: | Size: 13 KiB |
After Width: | Height: | Size: 91 KiB |
After Width: | Height: | Size: 54 KiB |
After Width: | Height: | Size: 10 KiB |
After Width: | Height: | Size: 8.7 KiB |
After Width: | Height: | Size: 43 KiB |
After Width: | Height: | Size: 26 KiB |
After Width: | Height: | Size: 32 KiB |
After Width: | Height: | Size: 23 KiB |
After Width: | Height: | Size: 24 KiB |
After Width: | Height: | Size: 6.9 KiB |
After Width: | Height: | Size: 40 KiB |
After Width: | Height: | Size: 31 KiB |
After Width: | Height: | Size: 3.5 KiB |
After Width: | Height: | Size: 34 KiB |
After Width: | Height: | Size: 2.4 KiB |
After Width: | Height: | Size: 50 KiB |
After Width: | Height: | Size: 108 KiB |
After Width: | Height: | Size: 7.6 KiB |
After Width: | Height: | Size: 22 KiB |
After Width: | Height: | Size: 20 KiB |
After Width: | Height: | Size: 33 KiB |
After Width: | Height: | Size: 46 KiB |
After Width: | Height: | Size: 15 KiB |
After Width: | Height: | Size: 2.2 MiB |
After Width: | Height: | Size: 97 KiB |
Before Width: | Height: | Size: 98 KiB After Width: | Height: | Size: 5.1 MiB |
Before Width: | Height: | Size: 19 KiB After Width: | Height: | Size: 115 KiB |
|
@ -173,6 +173,12 @@ const networks = [
|
||||||
url: 'https://www.realitykings.com',
|
url: 'https://www.realitykings.com',
|
||||||
description: 'Home of HD reality porn featuring the nicest tits and ass online! The hottest curvy girls in real amateur sex stories are only on REALITYkings.com',
|
description: 'Home of HD reality porn featuring the nicest tits and ass online! The hottest curvy girls in real amateur sex stories are only on REALITYkings.com',
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
slug: 'score',
|
||||||
|
name: 'SCORE',
|
||||||
|
url: 'https://www.scorepass.com',
|
||||||
|
description: '',
|
||||||
|
},
|
||||||
{
|
{
|
||||||
slug: 'teamskeet',
|
slug: 'teamskeet',
|
||||||
name: 'Team Skeet',
|
name: 'Team Skeet',
|
||||||
|
|
1490
seeds/01_sites.js
|
@ -4,9 +4,9 @@ const tagPosters = Object.entries({
|
||||||
'anal-creampie': [0, 'Gina Valentina and Jane Wilde in "A Very Special Anniversary" for Tushy'],
|
'anal-creampie': [0, 'Gina Valentina and Jane Wilde in "A Very Special Anniversary" for Tushy'],
|
||||||
'ass-to-mouth': ['poster', 'Alysa Gap and Logan in "Anal Buffet 4" for Evil Angel'],
|
'ass-to-mouth': ['poster', 'Alysa Gap and Logan in "Anal Buffet 4" for Evil Angel'],
|
||||||
'da-tp': [0, 'Natasha Teen in LegalPorno SZ2164'],
|
'da-tp': [0, 'Natasha Teen in LegalPorno SZ2164'],
|
||||||
'double-anal': [2, 'Lana Rhoades in "Gangbang Me 3" for HardX'],
|
'double-anal': [5, 'Riley Reid in "The Gangbang of Riley Reid" for Jules Jordan'],
|
||||||
'double-penetration': ['poster', 'Mia Malkova in "DP!" for HardX'],
|
'double-penetration': ['poster', 'Mia Malkova in "DP!" for HardX'],
|
||||||
'double-vaginal': [0, 'Aaliyah Hadid in "Squirting From Double Penetration With Anal" for Bang Bros'],
|
'double-vaginal': ['poster', 'Riley Reid in "Pizza That Ass" for Reid My Lips'],
|
||||||
'dv-tp': ['poster', 'Juelz Ventura in "Gangbanged 5" for Elegant Angel'],
|
'dv-tp': ['poster', 'Juelz Ventura in "Gangbanged 5" for Elegant Angel'],
|
||||||
'oral-creampie': [1, 'Keisha Grey in Brazzers House'],
|
'oral-creampie': [1, 'Keisha Grey in Brazzers House'],
|
||||||
'triple-anal': ['poster', 'Kristy Black in SZ1986 for LegalPorno'],
|
'triple-anal': ['poster', 'Kristy Black in SZ1986 for LegalPorno'],
|
||||||
|
@ -50,11 +50,12 @@ const tagPhotos = [
|
||||||
['da-tp', 2, 'Angel Smalls in GIO408 for LegalPorno'],
|
['da-tp', 2, 'Angel Smalls in GIO408 for LegalPorno'],
|
||||||
['da-tp', 3, 'Evelina Darling in GIO294'],
|
['da-tp', 3, 'Evelina Darling in GIO294'],
|
||||||
['da-tp', 4, 'Ninel Mojado aka Mira Cuckold in GIO063 for LegalPorno'],
|
['da-tp', 4, 'Ninel Mojado aka Mira Cuckold in GIO063 for LegalPorno'],
|
||||||
|
['double-anal', 2, 'Lana Rhoades in "Gangbang Me 3" for HardX'],
|
||||||
['double-anal', 'poster', 'Haley Reed in "Young Hot Ass" for Evil Angel'],
|
['double-anal', 'poster', 'Haley Reed in "Young Hot Ass" for Evil Angel'],
|
||||||
['double-anal', 0, 'Nicole Black doing double anal during a gangbang in GIO971 for LegalPorno'],
|
['double-anal', 0, 'Nicole Black doing double anal during a gangbang in GIO971 for LegalPorno'],
|
||||||
['double-anal', 1, 'Ria Sunn in SZ1801 for LegalPorno'],
|
['double-anal', 1, 'Ria Sunn in SZ1801 for LegalPorno'],
|
||||||
['double-penetration', 0],
|
['double-penetration', 0],
|
||||||
['double-vaginal', 'poster', 'Riley Reid in "Pizza That Ass" for Reid My Lips'],
|
['double-vaginal', 0, 'Aaliyah Hadid in "Squirting From Double Penetration With Anal" for Bang Bros'],
|
||||||
['dv-tp', 1, 'Adriana Chechik in "Adriana\'s Triple Anal Penetration!"'],
|
['dv-tp', 1, 'Adriana Chechik in "Adriana\'s Triple Anal Penetration!"'],
|
||||||
['dv-tp', 0, 'Luna Rival in LegalPorno SZ1490'],
|
['dv-tp', 0, 'Luna Rival in LegalPorno SZ1490'],
|
||||||
['facefucking', '0', 'Brea for Young Throats'],
|
['facefucking', '0', 'Brea for Young Throats'],
|
||||||
|
|
|
@ -346,7 +346,7 @@ async function scrapeActors(actorNames) {
|
||||||
const profileScrapers = [].concat(source).map(slug => ({ scraperSlug: slug, scraper: scrapers.actors[slug] }));
|
const profileScrapers = [].concat(source).map(slug => ({ scraperSlug: slug, scraper: scrapers.actors[slug] }));
|
||||||
|
|
||||||
try {
|
try {
|
||||||
return profileScrapers.reduce(async (outcome, { scraper, scraperSlug }) => outcome.catch(async () => {
|
return await profileScrapers.reduce(async (outcome, { scraper, scraperSlug }) => outcome.catch(async () => {
|
||||||
logger.verbose(`Searching '${actorName}' on ${scraperSlug}`);
|
logger.verbose(`Searching '${actorName}' on ${scraperSlug}`);
|
||||||
|
|
||||||
const profile = await scraper.fetchProfile(actorEntry ? actorEntry.name : actorName, scraperSlug);
|
const profile = await scraper.fetchProfile(actorEntry ? actorEntry.name : actorName, scraperSlug);
|
||||||
|
@ -365,7 +365,7 @@ async function scrapeActors(actorNames) {
|
||||||
throw new Error(`Profile for ${actorName} not available on ${scraperSlug}`);
|
throw new Error(`Profile for ${actorName} not available on ${scraperSlug}`);
|
||||||
}), Promise.reject(new Error()));
|
}), Promise.reject(new Error()));
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error(error);
|
logger.warn(`Error in scraper ${source}: ${error.message}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
|
@ -418,6 +418,7 @@ async function scrapeActors(actorNames) {
|
||||||
|
|
||||||
return profile;
|
return profile;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
console.log(error);
|
||||||
logger.warn(`${actorName}: ${error}`);
|
logger.warn(`${actorName}: ${error}`);
|
||||||
|
|
||||||
return null;
|
return null;
|
||||||
|
|
12
src/media.js
|
@ -188,7 +188,7 @@ async function storePhotos(photos, {
|
||||||
targetId,
|
targetId,
|
||||||
subpath,
|
subpath,
|
||||||
primaryRole, // role to assign to first photo if not already in database, used mainly for avatars
|
primaryRole, // role to assign to first photo if not already in database, used mainly for avatars
|
||||||
entropy = 2.5, // filter out fallback avatars and other generic clipart
|
entropyFilter = 2.5, // filter out fallback avatars and other generic clipart
|
||||||
}, label) {
|
}, label) {
|
||||||
if (!photos || photos.length === 0) {
|
if (!photos || photos.length === 0) {
|
||||||
logger.info(`No ${role}s available for ${label}`);
|
logger.info(`No ${role}s available for ${label}`);
|
||||||
|
@ -200,7 +200,7 @@ async function storePhotos(photos, {
|
||||||
|
|
||||||
const metaFiles = await Promise.map(sourceOriginals, async (photoUrl, index) => fetchPhoto(photoUrl, index, label), {
|
const metaFiles = await Promise.map(sourceOriginals, async (photoUrl, index) => fetchPhoto(photoUrl, index, label), {
|
||||||
concurrency: 10,
|
concurrency: 10,
|
||||||
}).filter(photo => photo && photo.entropy > entropy);
|
}).filter(photo => photo && photo.entropy > entropyFilter);
|
||||||
|
|
||||||
const metaFilesByHash = metaFiles.reduce((acc, photo) => ({ ...acc, [photo.hash]: photo }), {}); // pre-filter hash duplicates within set; may occur through fallbacks
|
const metaFilesByHash = metaFiles.reduce((acc, photo) => ({ ...acc, [photo.hash]: photo }), {}); // pre-filter hash duplicates within set; may occur through fallbacks
|
||||||
const [hashDuplicates, hashOriginals] = await findDuplicates(Object.values(metaFilesByHash), 'hash', 'hash', label);
|
const [hashDuplicates, hashOriginals] = await findDuplicates(Object.values(metaFilesByHash), 'hash', 'hash', label);
|
||||||
|
@ -285,6 +285,7 @@ async function storeReleasePhotos(releases, label) {
|
||||||
|
|
||||||
async function storeTrailer(trailers, {
|
async function storeTrailer(trailers, {
|
||||||
domain = 'releases',
|
domain = 'releases',
|
||||||
|
role = 'trailer',
|
||||||
targetId,
|
targetId,
|
||||||
subpath,
|
subpath,
|
||||||
}, label) {
|
}, label) {
|
||||||
|
@ -294,7 +295,7 @@ async function storeTrailer(trailers, {
|
||||||
: trailers;
|
: trailers;
|
||||||
|
|
||||||
if (!trailer || !trailer.src) {
|
if (!trailer || !trailer.src) {
|
||||||
logger.info(`No trailer available for ${label}`);
|
logger.info(`No ${role} available for ${label}`);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -306,7 +307,7 @@ async function storeTrailer(trailers, {
|
||||||
|
|
||||||
const res = await bhttp.get(trailerX.src);
|
const res = await bhttp.get(trailerX.src);
|
||||||
const hash = getHash(res.body);
|
const hash = getHash(res.body);
|
||||||
const filepath = path.join(domain, subpath, `trailer${trailerX.quality ? `_${trailerX.quality}` : ''}.${mime.getExtension(mimetype)}`);
|
const filepath = path.join(domain, subpath, `${role}${trailerX.quality ? `_${trailerX.quality}` : ''}.${mime.getExtension(mimetype)}`);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
trailer: res.body,
|
trailer: res.body,
|
||||||
|
@ -327,6 +328,7 @@ async function storeTrailer(trailers, {
|
||||||
source: trailerX.source,
|
source: trailerX.source,
|
||||||
quality: trailerX.quality,
|
quality: trailerX.quality,
|
||||||
hash: trailerX.hash,
|
hash: trailerX.hash,
|
||||||
|
type: role,
|
||||||
})))
|
})))
|
||||||
.returning('*');
|
.returning('*');
|
||||||
|
|
||||||
|
@ -336,7 +338,7 @@ async function storeTrailer(trailers, {
|
||||||
? [...sourceDuplicates, ...hashDuplicates, ...newTrailers]
|
? [...sourceDuplicates, ...hashDuplicates, ...newTrailers]
|
||||||
: [...sourceDuplicates, ...hashDuplicates];
|
: [...sourceDuplicates, ...hashDuplicates];
|
||||||
|
|
||||||
await upsert('releases_trailers', trailerEntries.map(trailerEntry => ({
|
await upsert(`releases_${role}s`, trailerEntries.map(trailerEntry => ({
|
||||||
release_id: targetId,
|
release_id: targetId,
|
||||||
media_id: trailerEntry.id,
|
media_id: trailerEntry.id,
|
||||||
})), ['release_id', 'media_id']);
|
})), ['release_id', 'media_id']);
|
||||||
|
|
|
@ -345,6 +345,13 @@ async function storeReleaseAssets(releases) {
|
||||||
await storeTrailer(release.trailer, {
|
await storeTrailer(release.trailer, {
|
||||||
targetId: release.id,
|
targetId: release.id,
|
||||||
subpath,
|
subpath,
|
||||||
|
role: 'trailer',
|
||||||
|
}, identifier);
|
||||||
|
|
||||||
|
await storeTrailer(release.teaser, {
|
||||||
|
targetId: release.id,
|
||||||
|
subpath,
|
||||||
|
role: 'teaser',
|
||||||
}, identifier);
|
}, identifier);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error(error.message);
|
logger.error(error.message);
|
||||||
|
|
|
@ -5,7 +5,7 @@ const bhttp = require('bhttp');
|
||||||
const { ex } = require('../utils/q');
|
const { ex } = require('../utils/q');
|
||||||
|
|
||||||
function scrapeProfile(html) {
|
function scrapeProfile(html) {
|
||||||
const { q, qa, qd, qi, qu } = ex(html); /* eslint-disable-line object-curly-newline */
|
const { q, qa, qd, qi, qus } = ex(html); /* eslint-disable-line object-curly-newline */
|
||||||
const profile = {};
|
const profile = {};
|
||||||
|
|
||||||
const bio = qa('.infobox tr[valign="top"]')
|
const bio = qa('.infobox tr[valign="top"]')
|
||||||
|
@ -59,19 +59,15 @@ function scrapeProfile(html) {
|
||||||
if (bio.Blood_group) profile.blood = bio.Blood_group;
|
if (bio.Blood_group) profile.blood = bio.Blood_group;
|
||||||
if (bio.Also_known_as) profile.aliases = bio.Also_known_as.split(', ');
|
if (bio.Also_known_as) profile.aliases = bio.Also_known_as.split(', ');
|
||||||
|
|
||||||
const avatars = qi('.image img');
|
const avatarThumbPath = qi('.image img');
|
||||||
|
|
||||||
if (avatars.length > 0) {
|
if (avatarThumbPath && !/NoImageAvailable/.test(avatarThumbPath)) {
|
||||||
const [avatarThumbPath] = avatars;
|
|
||||||
|
|
||||||
if (!/NoImageAvailable/.test(avatarThumbPath)) {
|
|
||||||
const avatarPath = avatarThumbPath.slice(0, avatarThumbPath.lastIndexOf('/')).replace('thumb/', '');
|
const avatarPath = avatarThumbPath.slice(0, avatarThumbPath.lastIndexOf('/')).replace('thumb/', '');
|
||||||
|
|
||||||
profile.avatar = `http://www.boobpedia.com${avatarPath}`;
|
profile.avatar = `http://www.boobpedia.com${avatarPath}`;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
profile.social = qu('.infobox a.external');
|
profile.social = qus('.infobox a.external');
|
||||||
|
|
||||||
return profile;
|
return profile;
|
||||||
}
|
}
|
||||||
|
|
|
@ -101,7 +101,7 @@ function scrapeScene(html, url, site) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function scrapeProfile(html) {
|
function scrapeProfile(html) {
|
||||||
const { q, qu } = ex(html);
|
const { q, qus } = ex(html);
|
||||||
const profile = {};
|
const profile = {};
|
||||||
|
|
||||||
profile.description = q('.bio_about_text', true);
|
profile.description = q('.bio_about_text', true);
|
||||||
|
@ -109,7 +109,7 @@ function scrapeProfile(html) {
|
||||||
const avatar = q('img.performer-pic', 'src');
|
const avatar = q('img.performer-pic', 'src');
|
||||||
if (avatar) profile.avatar = `https:${avatar}`;
|
if (avatar) profile.avatar = `https:${avatar}`;
|
||||||
|
|
||||||
profile.releases = qu('.scene-item > a:first-child');
|
profile.releases = qus('.scene-item > a:first-child');
|
||||||
|
|
||||||
return profile;
|
return profile;
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,206 @@
|
||||||
|
'use strict';
|
||||||
|
|
||||||
|
const bhttp = require('bhttp');
|
||||||
|
|
||||||
|
const { ex, exa } = require('../utils/q');
|
||||||
|
const slugify = require('../utils/slugify');
|
||||||
|
const { heightToCm, lbsToKg } = require('../utils/convert');
|
||||||
|
|
||||||
|
function scrapePhotos(html) {
|
||||||
|
const { qis } = ex(html, '#photos-page');
|
||||||
|
const photos = qis('img');
|
||||||
|
|
||||||
|
return photos.map(photo => [
|
||||||
|
photo
|
||||||
|
.replace('x_800', 'x_xl')
|
||||||
|
.replace('_tn', ''),
|
||||||
|
photo,
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchPhotos(url) {
|
||||||
|
const res = await bhttp.get(url);
|
||||||
|
|
||||||
|
if (res.statusCode === 200) {
|
||||||
|
return scrapePhotos(res.body.toString(), url);
|
||||||
|
}
|
||||||
|
|
||||||
|
return [];
|
||||||
|
}
|
||||||
|
|
||||||
|
function scrapeAll(html) {
|
||||||
|
return exa(html, '.container .video').map(({ q, qa, qd, ql }) => {
|
||||||
|
const release = {};
|
||||||
|
|
||||||
|
const linkEl = q('a.i-title');
|
||||||
|
|
||||||
|
release.title = linkEl.textContent.trim();
|
||||||
|
|
||||||
|
const url = new URL(linkEl.href);
|
||||||
|
release.url = `${url.origin}${url.pathname}`;
|
||||||
|
|
||||||
|
// this is a photo album, not a scene (used for profiles)
|
||||||
|
if (/photos\//.test(url)) return null;
|
||||||
|
|
||||||
|
[release.entryId] = url.pathname.split('/').slice(-2);
|
||||||
|
|
||||||
|
release.date = qd('.i-date', 'MMM DD', /\w+ \d{1,2}$/);
|
||||||
|
release.actors = qa('.i-model', true);
|
||||||
|
release.duration = ql('.i-amount');
|
||||||
|
|
||||||
|
const posterEl = q('.item-img img');
|
||||||
|
|
||||||
|
if (posterEl) {
|
||||||
|
release.poster = `https:${posterEl.src}`;
|
||||||
|
release.teaser = {
|
||||||
|
src: `https:${posterEl.dataset.gifPreview}`,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
return release;
|
||||||
|
}).filter(Boolean);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function scrapeScene(html, url) {
|
||||||
|
const { q, qa, qd, ql, qu, qp, qt } = ex(html, '#videos-page');
|
||||||
|
const release = {};
|
||||||
|
|
||||||
|
[release.entryId] = new URL(url).pathname.split('/').slice(-2);
|
||||||
|
|
||||||
|
release.title = q('#breadcrumb-top + h1', true);
|
||||||
|
release.description = q('.p-desc', true);
|
||||||
|
|
||||||
|
release.actors = qa('a[href*=models]', true);
|
||||||
|
release.tags = qa('a[href*=tag]', true);
|
||||||
|
|
||||||
|
const dateEl = qa('.value').find(el => /\w+ \d+\w+, \d{4}/.test(el.textContent));
|
||||||
|
release.date = qd(dateEl, null, 'MMMM Do, YYYY');
|
||||||
|
|
||||||
|
const durationEl = qa('value').find(el => /\d{1,3}:\d{2}/.test(el.textContent));
|
||||||
|
release.duration = ql(durationEl);
|
||||||
|
|
||||||
|
const photosUrl = qu('a[href*=photos]');
|
||||||
|
release.photos = await fetchPhotos(photosUrl);
|
||||||
|
release.poster = qp('video'); // _800.jpg is larger than _xl.jpg in landscape
|
||||||
|
|
||||||
|
const trailer = qt();
|
||||||
|
release.trailer = [
|
||||||
|
{
|
||||||
|
// don't rely on trailer always being 720p by default
|
||||||
|
src: trailer.replace(/\d+p\.mp4/, '720p.mp4'),
|
||||||
|
quality: 720,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
src: trailer.replace(/\d+p\.mp4/, '360p.mp4'),
|
||||||
|
quality: 360,
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const stars = q('.rate-box').dataset.score;
|
||||||
|
if (stars) release.rating = { stars };
|
||||||
|
|
||||||
|
return release;
|
||||||
|
}
|
||||||
|
|
||||||
|
function scrapeModels(html, actorName) {
|
||||||
|
const { qa } = ex(html);
|
||||||
|
const model = qa('.model a').find(link => link.title === actorName);
|
||||||
|
|
||||||
|
return model?.href || null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function scrapeProfile(html) {
|
||||||
|
const { q, qa, qi } = ex(html, '#model-page');
|
||||||
|
const profile = { gender: 'female' };
|
||||||
|
|
||||||
|
const bio = qa('.stat').reduce((acc, el) => {
|
||||||
|
const prop = q(el, '.label', true).slice(0, -1);
|
||||||
|
const key = slugify(prop, false, '_');
|
||||||
|
const value = q(el, '.value', true);
|
||||||
|
|
||||||
|
return {
|
||||||
|
...acc,
|
||||||
|
[key]: value,
|
||||||
|
};
|
||||||
|
}, {});
|
||||||
|
|
||||||
|
if (bio.location) profile.residencePlace = bio.location.replace('Czech Repulic', 'Czech Republic'); // see Laura Lion
|
||||||
|
|
||||||
|
if (bio.birthday) {
|
||||||
|
const birthMonth = bio.birthday.match(/^\w+/)[0].toLowerCase();
|
||||||
|
const [birthDay] = bio.birthday.match(/\d+/);
|
||||||
|
|
||||||
|
profile.birthday = [birthMonth, birthDay]; // currently unused, not to be confused with birthdate
|
||||||
|
}
|
||||||
|
|
||||||
|
if (bio.ethnicity) profile.ethnicity = bio.ethnicity;
|
||||||
|
if (bio.hair_color) profile.hair = bio.hair_color;
|
||||||
|
|
||||||
|
if (bio.height) profile.height = heightToCm(bio.height);
|
||||||
|
if (bio.weight) profile.weight = lbsToKg(bio.weight);
|
||||||
|
|
||||||
|
if (bio.bra_size) profile.bust = bio.bra_size;
|
||||||
|
if (bio.measurements) [, profile.waist, profile.hip] = bio.measurements.split('-');
|
||||||
|
|
||||||
|
if (bio.occupation) profile.occupation = bio.occupation;
|
||||||
|
|
||||||
|
const avatar = qi('img');
|
||||||
|
if (avatar) profile.avatar = avatar;
|
||||||
|
|
||||||
|
const releases = ex(html, '#model-page + .container');
|
||||||
|
profile.releases = scrapeAll(releases.document.outerHTML);
|
||||||
|
|
||||||
|
return profile;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchLatest(site, page = 1) {
|
||||||
|
const url = `${site.url}/big-boob-videos?page=${page}`;
|
||||||
|
const res = await bhttp.get(url);
|
||||||
|
|
||||||
|
if (res.statusCode === 200) {
|
||||||
|
return scrapeAll(res.body.toString(), site);
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchScene(url, site) {
|
||||||
|
const res = await bhttp.get(url);
|
||||||
|
|
||||||
|
if (res.statusCode === 200) {
|
||||||
|
return scrapeScene(res.body.toString(), url, site);
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchProfile(actorName, scraperSlug, page = 1) {
|
||||||
|
const letter = actorName.charAt(0).toUpperCase();
|
||||||
|
|
||||||
|
const url = `https://www.scoreland.com/big-boob-models/browse/${letter}/?page=${page}`;
|
||||||
|
const res = await bhttp.get(url);
|
||||||
|
|
||||||
|
if (res.statusCode === 200) {
|
||||||
|
const actorUrl = scrapeModels(res.body.toString(), actorName);
|
||||||
|
|
||||||
|
if (actorUrl) {
|
||||||
|
const actorRes = await bhttp.get(actorUrl);
|
||||||
|
|
||||||
|
if (actorRes.statusCode === 200) {
|
||||||
|
return scrapeProfile(actorRes.body.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return fetchProfile(actorName, scraperSlug, page + 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
module.exports = {
|
||||||
|
fetchLatest,
|
||||||
|
fetchScene,
|
||||||
|
fetchProfile,
|
||||||
|
};
|
|
@ -34,6 +34,7 @@ const men = require('./men');
|
||||||
const metrohd = require('./metrohd');
|
const metrohd = require('./metrohd');
|
||||||
const mofos = require('./mofos');
|
const mofos = require('./mofos');
|
||||||
const naughtyamerica = require('./naughtyamerica');
|
const naughtyamerica = require('./naughtyamerica');
|
||||||
|
const score = require('./score');
|
||||||
const twentyonesextury = require('./21sextury');
|
const twentyonesextury = require('./21sextury');
|
||||||
const xempire = require('./xempire');
|
const xempire = require('./xempire');
|
||||||
const wicked = require('./wicked');
|
const wicked = require('./wicked');
|
||||||
|
@ -78,6 +79,7 @@ module.exports = {
|
||||||
puretaboo,
|
puretaboo,
|
||||||
naughtyamerica,
|
naughtyamerica,
|
||||||
realitykings,
|
realitykings,
|
||||||
|
score,
|
||||||
teamskeet,
|
teamskeet,
|
||||||
vixen,
|
vixen,
|
||||||
vogov,
|
vogov,
|
||||||
|
@ -109,6 +111,7 @@ module.exports = {
|
||||||
naughtyamerica,
|
naughtyamerica,
|
||||||
pornhub,
|
pornhub,
|
||||||
realitykings,
|
realitykings,
|
||||||
|
score,
|
||||||
transangels,
|
transangels,
|
||||||
wicked,
|
wicked,
|
||||||
xempire,
|
xempire,
|
||||||
|
|
|
@ -116,7 +116,7 @@ function scrapeLatest(html) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function scrapeScene(html, url) {
|
function scrapeScene(html, url) {
|
||||||
const { q, qa, qd, qu, ql, qm } = ex(html);
|
const { q, qa, qd, qus, ql, qm } = ex(html);
|
||||||
const release = { url };
|
const release = { url };
|
||||||
|
|
||||||
// release.entryId = slugify(release.title);
|
// release.entryId = slugify(release.title);
|
||||||
|
@ -131,7 +131,7 @@ function scrapeScene(html, url) {
|
||||||
release.actors = qa('.info-video-models a', true);
|
release.actors = qa('.info-video-models a', true);
|
||||||
release.tags = qa('.info-video-category a', true);
|
release.tags = qa('.info-video-category a', true);
|
||||||
|
|
||||||
release.photos = qu('.swiper-wrapper .swiper-slide a').map(source => source.replace('.jpg/', '.jpg'));
|
release.photos = qus('.swiper-wrapper .swiper-slide a').map(source => source.replace('.jpg/', '.jpg'));
|
||||||
release.poster = qm('meta[property="og:image"');
|
release.poster = qm('meta[property="og:image"');
|
||||||
|
|
||||||
if (!release.poster) {
|
if (!release.poster) {
|
||||||
|
|
|
@ -3,11 +3,21 @@
|
||||||
const { JSDOM } = require('jsdom');
|
const { JSDOM } = require('jsdom');
|
||||||
const moment = require('moment');
|
const moment = require('moment');
|
||||||
|
|
||||||
|
function prefixProtocol(url, protocol = 'https') {
|
||||||
|
if (protocol && /^\/\//.test(url)) {
|
||||||
|
return `${protocol}:${url}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
return url;
|
||||||
|
}
|
||||||
|
|
||||||
function q(context, selector, attrArg, trim = true) {
|
function q(context, selector, attrArg, trim = true) {
|
||||||
const attr = attrArg === true ? 'textContent' : attrArg;
|
const attr = attrArg === true ? 'textContent' : attrArg;
|
||||||
|
|
||||||
if (attr) {
|
if (attr) {
|
||||||
const value = context.querySelector(selector)?.[attr];
|
const value = selector
|
||||||
|
? context.querySelector(selector)?.[attr]
|
||||||
|
: context[attr];
|
||||||
|
|
||||||
return trim ? value?.trim() : value;
|
return trim ? value?.trim() : value;
|
||||||
}
|
}
|
||||||
|
@ -30,12 +40,14 @@ function qmeta(context, selector, attrArg = 'content', trim = true) {
|
||||||
}
|
}
|
||||||
|
|
||||||
function qdate(context, selector, format, match, attr = 'textContent') {
|
function qdate(context, selector, format, match, attr = 'textContent') {
|
||||||
const dateString = context.querySelector(selector)?.[attr];
|
const dateString = selector
|
||||||
|
? context.querySelector(selector)?.[attr]
|
||||||
|
: context[attr];
|
||||||
|
|
||||||
if (!dateString) return null;
|
if (!dateString) return null;
|
||||||
|
|
||||||
if (match) {
|
if (match) {
|
||||||
const dateStamp = dateString.match(match);
|
const dateStamp = dateString.trim().match(match);
|
||||||
|
|
||||||
if (dateStamp) return moment.utc(dateStamp[0], format).toDate();
|
if (dateStamp) return moment.utc(dateStamp[0], format).toDate();
|
||||||
return null;
|
return null;
|
||||||
|
@ -44,20 +56,41 @@ function qdate(context, selector, format, match, attr = 'textContent') {
|
||||||
return moment.utc(dateString.trim(), format).toDate();
|
return moment.utc(dateString.trim(), format).toDate();
|
||||||
}
|
}
|
||||||
|
|
||||||
function qimages(context, selector = 'img', attr = 'src') {
|
function qimage(context, selector = 'img', attr = 'src', protocol = 'https') {
|
||||||
return qall(context, selector, attr);
|
const image = q(context, selector, attr);
|
||||||
|
|
||||||
|
// no attribute means q output will be HTML element
|
||||||
|
return attr ? prefixProtocol(image, protocol) : image;
|
||||||
}
|
}
|
||||||
|
|
||||||
function qurls(context, selector = 'a', attr = 'href') {
|
function qimages(context, selector = 'img', attr = 'src', protocol = 'https') {
|
||||||
return qall(context, selector, attr);
|
const images = qall(context, selector, attr);
|
||||||
|
|
||||||
|
return attr ? images.map(image => prefixProtocol(image, protocol)) : images;
|
||||||
}
|
}
|
||||||
|
|
||||||
function qposter(context, selector = 'video', attr = 'poster') {
|
function qurl(context, selector = 'a', attr = 'href', protocol = 'https') {
|
||||||
return q(context, selector, attr);
|
const url = q(context, selector, attr);
|
||||||
|
|
||||||
|
return attr ? prefixProtocol(url, protocol) : url;
|
||||||
}
|
}
|
||||||
|
|
||||||
function qtrailer(context, selector = 'source', attr = 'src') {
|
function qurls(context, selector = 'a', attr = 'href', protocol = 'https') {
|
||||||
return q(context, selector, attr);
|
const urls = qall(context, selector, attr);
|
||||||
|
|
||||||
|
return attr ? urls.map(url => prefixProtocol(url, protocol)) : urls;
|
||||||
|
}
|
||||||
|
|
||||||
|
function qposter(context, selector = 'video', attr = 'poster', protocol = 'https') {
|
||||||
|
const poster = q(context, selector, attr);
|
||||||
|
|
||||||
|
return attr ? prefixProtocol(poster, protocol) : poster;
|
||||||
|
}
|
||||||
|
|
||||||
|
function qtrailer(context, selector = 'source', attr = 'src', protocol = 'https') {
|
||||||
|
const trailer = q(context, selector, attr);
|
||||||
|
|
||||||
|
return attr ? prefixProtocol(trailer, protocol) : trailer;
|
||||||
}
|
}
|
||||||
|
|
||||||
function qlength(context, selector, attr = 'textContent') {
|
function qlength(context, selector, attr = 'textContent') {
|
||||||
|
@ -77,20 +110,24 @@ const funcs = {
|
||||||
q,
|
q,
|
||||||
qall,
|
qall,
|
||||||
qdate,
|
qdate,
|
||||||
|
qimage,
|
||||||
qimages,
|
qimages,
|
||||||
qposter,
|
qposter,
|
||||||
qlength,
|
qlength,
|
||||||
qmeta,
|
qmeta,
|
||||||
qtrailer,
|
qtrailer,
|
||||||
qurls,
|
qurls,
|
||||||
|
qurl,
|
||||||
qa: qall,
|
qa: qall,
|
||||||
qd: qdate,
|
qd: qdate,
|
||||||
qi: qimages,
|
qi: qimage,
|
||||||
|
qis: qimages,
|
||||||
qp: qposter,
|
qp: qposter,
|
||||||
ql: qlength,
|
ql: qlength,
|
||||||
qm: qmeta,
|
qm: qmeta,
|
||||||
qt: qtrailer,
|
qt: qtrailer,
|
||||||
qu: qurls,
|
qu: qurl,
|
||||||
|
qus: qurls,
|
||||||
};
|
};
|
||||||
|
|
||||||
function ctx(element, window) {
|
function ctx(element, window) {
|
||||||
|
@ -110,18 +147,29 @@ function ctx(element, window) {
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function ctxa(context, selector) {
|
function ctxa(context, selector, window) {
|
||||||
return Array.from(context.querySelectorAll(selector)).map(element => ctx(element));
|
return Array.from(context.querySelectorAll(selector)).map(element => ctx(element, window));
|
||||||
}
|
}
|
||||||
|
|
||||||
function ex(html) {
|
function ex(html, selector) {
|
||||||
const { window } = new JSDOM(html);
|
const { window } = new JSDOM(html);
|
||||||
|
|
||||||
|
if (selector) {
|
||||||
|
return ctx(window.document.querySelector(selector), window);
|
||||||
|
}
|
||||||
|
|
||||||
return ctx(window.document, window);
|
return ctx(window.document, window);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function exa(html, selector) {
|
||||||
|
const { window } = new JSDOM(html);
|
||||||
|
|
||||||
|
return ctxa(window.document, selector, window);
|
||||||
|
}
|
||||||
|
|
||||||
module.exports = {
|
module.exports = {
|
||||||
ex,
|
ex,
|
||||||
|
exa,
|
||||||
ctx,
|
ctx,
|
||||||
ctxa,
|
ctxa,
|
||||||
...funcs,
|
...funcs,
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
function slugify(string, encode = false) {
|
function slugify(string, encode = false, delimiter = '-') {
|
||||||
const slug = string.trim().toLowerCase().match(/\w+/g).join('-');
|
const slug = string.trim().toLowerCase().match(/\w+/g).join(delimiter);
|
||||||
|
|
||||||
return encode ? encodeURI(slug) : slug;
|
return encode ? encodeURI(slug) : slug;
|
||||||
}
|
}
|
||||||
|
|