Added First Anal Quest and Double View Casting latest and scene scraper.

This commit is contained in:
DebaucheryLibrarian 2020-12-02 03:17:32 +01:00
parent bfbfa761ef
commit 2656e3adb0
33 changed files with 166 additions and 8 deletions

Binary file not shown.

After

Width:  |  Height:  |  Size: 98 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 74 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 37 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 13 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 98 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 76 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 74 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 37 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 33 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 29 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 74 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 106 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 74 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 26 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 34 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 74 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 30 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 29 KiB

View File

@ -224,6 +224,11 @@ const networks = [
description: 'Wherever they go, there is porn. Hospital, Taxis, Casting… Maybe fucking to a fake cop, fake agent or fake taxi driver. And we record it all.',
parent: 'mindgeek',
},
{
slug: 'firstanalquest',
name: 'First Anal Quest',
url: 'http://www.firstanalquest.com',
},
{
slug: 'forbondage',
name: 'ForBondage',

View File

@ -2757,6 +2757,55 @@ const sites = [
url: 'https://afterhoursexposed.com',
parent: 'fcuk',
},
// FIRST ANAL QUEST
{
slug: 'firstanalquest',
name: 'First Anal Quest',
url: 'http://www.firstanalquest.com',
tags: ['anal'],
parent: 'firstanalquest',
parameters: {
layout: 'a',
latest: 'http://www.firstanalquest.com/latest-updates',
},
},
{
slug: 'doubleviewcasting',
name: 'Double View Casting',
url: 'http://www.doubleviewcasting.com',
parent: 'firstanalquest',
parameters: {
layout: 'a',
latest: 'http://www.doubleviewcasting.com/scenes/page',
},
},
{
slug: 'fuckndrive',
name: 'Fuck\'n\'Drive',
url: 'http://www.fuckndrive.com',
parent: 'firstanalquest',
parameters: {
layout: 'b',
},
},
{
slug: 'jizzonteens',
name: 'Jizz On Teens',
url: 'http://www.jizzonteens.com',
parent: 'firstanalquest',
parameters: {
layout: 'b',
},
},
{
slug: 'wantedgfs',
name: 'Wanted GFs',
url: 'http://www.wantedgfs.com',
parent: 'firstanalquest',
parameters: {
layout: 'c',
},
},
// FOR BONDAGE
{
name: 'Crowd Bondage',

View File

@ -41,7 +41,7 @@ async function findEntities(baseReleases) {
.orderBy('entities.type', 'asc');
// channel entity will overwrite network entity
const entitiesBySlug = entities.reduce((accEntities, entity) => ({ ...accEntities, [entity.slug]: entity }), {});
const entitiesBySlug = entities.reduce((accEntities, entity) => ({ ...accEntities, [entity.slug]: accEntities[entity.slug] || entity }), {});
return entitiesBySlug;
}
@ -102,13 +102,14 @@ async function scrapeRelease(baseRelease, entities, type = 'scene') {
}
const scraper = scrapers.releases[entity.slug] || scrapers.releases[entity.parent?.slug];
const layoutScraper = scraper[entity.parameters?.layout] || scraper;
if (!scraper) {
if (!layoutScraper) {
logger.warn(`Could not find scraper for ${baseRelease.url}`);
return baseRelease;
}
if ((type === 'scene' && !scraper.fetchScene) || (type === 'movie' && !scraper.fetchMovie)) {
if ((type === 'scene' && !layoutScraper.fetchScene) || (type === 'movie' && !layoutScraper.fetchMovie)) {
logger.warn(`The '${entity.name}'-scraper cannot fetch individual ${type}s`);
return baseRelease;
}
@ -117,8 +118,8 @@ async function scrapeRelease(baseRelease, entities, type = 'scene') {
logger.verbose(`Fetching ${type} ${baseRelease.url}`);
const scrapedRelease = type === 'scene'
? await scraper.fetchScene(baseRelease.url, entity, baseRelease, include, null)
: await scraper.fetchMovie(baseRelease.url, entity, baseRelease, include, null);
? await layoutScraper.fetchScene(baseRelease.url, entity, baseRelease, include, null)
: await layoutScraper.fetchMovie(baseRelease.url, entity, baseRelease, include, null);
const mergedRelease = {
...baseRelease,

View File

@ -0,0 +1,99 @@
'use strict';
const qu = require('../utils/qu');
function scrapeAllA(scenes, channel) {
return scenes.map(({ query }) => {
const release = {};
release.url = query.url('a.thumb-img, a.thumb', 'href', { origin: channel.url });
release.entryId = new URL(release.url).pathname.match(/(\d+)\/?$/)?.[1];
release.title = query.text('.thumb-title, .title');
release.date = query.date('.thumb-added, .date', ['MMM D, YYYY', 'MMMM DD, YYYY'], /\w+ \d{1,2}, \d{4}/);
release.duration = query.dur('.thumb-duration');
release.actors = query.all('.thumb-models a, .models a').map(actorEl => ({
name: query.cnt(actorEl),
url: query.url(actorEl, null, 'href', { origin: channel.url }),
}));
const [, photoUrl, photoCount] = query.q('.thumb-img img', 'onmouseover')?.match(/'(.*)', (\d+)\)/) || [];
if (photoUrl && photoCount) {
[release.poster, ...release.photos] = Array.from({ length: 5 }, (value, index) => `${photoUrl}${index + 1}.jpg`);
} else {
release.poster = query.img('.thumb-img img, .thumb img', 'src', { origin: channel.url });
}
release.tags = query.cnts('.tags a');
release.rating = query.number('.thumb-rating');
console.log(release);
return release;
});
}
function scrapeSceneA({ query }, url, channel) {
const release = {};
release.entryId = new URL(url).pathname.match(/(\d+)\/?$/)?.[1];
release.title = query.cnt('.title, .scene-title h3').replace(/:$/, '');
release.description = query.cnt('.text-desc p, .info-description p');
release.duration = query.dur('.media-body li span, .duration');
release.actors = query.all('.media-body a[href*="models/"], .models a').map(actorEl => ({
name: query.cnt(actorEl),
url: query.url(actorEl, null, 'href', { origin: channel.url }),
}));
release.tags = query.cnts('.media-body a[href*="tags/"], .tags a');
release.poster = [
query.img('.player-preview'),
qu.prefixUrl(`/contents/videos_screenshots/0/${release.entryId}/preview_trailer.mp4.jpg`, channel.url),
qu.prefixUrl(query.q('param[name="flashvars"]', 'value')?.match(/poster=(.*\.jpg)/)?.[1], channel.url),
qu.prefixUrl(`/contents/scenes/${release.entyId}/thumbnails/920x518.jpg`, channel.url),
];
release.photos = query.urls('.thumb-album a:not([href="#"]), .thumbs-photo a:not([href*="signup"])', 'href', { origin: channel.url })
.concat(query.imgs('.thumb-album a[href="#"] img, .thumbs-photo a[href*="signup"] img', 'src', { origin: channel.url }));
release.trailer = query.url('a[href*="get_file/"], .download a');
console.log(release);
return release;
}
async function fetchLatestA(channel, page) {
const url = channel.parameters?.latest
? `${channel.parameters.latest}/${page}`
: `${channel.url}/latest-updates/${page}/`;
const res = await qu.getAll(url, '.list-thumbs ul > li, .main-thumbs > li');
if (res.ok) {
return scrapeAllA(res.items, channel);
}
return res.status;
}
async function fetchSceneA(url, channel) {
const res = await qu.get(url, '.main, .main-content');
if (res.ok) {
return scrapeSceneA(res.item, url, channel);
}
return res.status;
}
module.exports = {
a: {
fetchLatest: fetchLatestA,
fetchScene: fetchSceneA,
},
};

View File

@ -22,6 +22,7 @@ const evilangel = require('./evilangel');
const fakehub = require('./fakehub');
const famedigital = require('./famedigital');
const fantasymassage = require('./fantasymassage');
const firstanalquest = require('./firstanalquest');
const fcuk = require('./fcuk');
const fullpornnetwork = require('./fullpornnetwork');
const girlsway = require('./girlsway');
@ -110,6 +111,7 @@ const scrapers = {
famedigital,
fantasymassage,
fcuk,
firstanalquest,
forbondage: porndoe,
fullpornnetwork,
girlsway,

View File

@ -209,7 +209,9 @@ async function scrapeChannel(channelEntity, accNetworkReleases) {
|| scrapers.releases[channelEntity.parent?.slug]
|| scrapers.releases[channelEntity.parent?.parent?.slug];
if (!scraper) {
const layoutScraper = scraper?.[channelEntity.parameters?.layout] || scraper;
if (!layoutScraper) {
logger.warn(`No scraper found for '${channelEntity.name}' (${channelEntity.parent?.name})`);
return emptyReleases;
}
@ -217,7 +219,7 @@ async function scrapeChannel(channelEntity, accNetworkReleases) {
try {
const beforeFetchLatest = await scraper.beforeFetchLatest?.(channelEntity);
return await scrapeChannelReleases(scraper, channelEntity, {
return await scrapeChannelReleases(layoutScraper, channelEntity, {
...accNetworkReleases,
beforeFetchLatest,
});

View File

@ -161,7 +161,7 @@ function styles(context, selector, styleAttr) {
return elStyles;
}
function number(context, selector, match = /\d+/, attr = 'textContent') {
function number(context, selector, match = /\d+(\.\d*)?/, attr = 'textContent') {
const value = q(context, selector, attr);
if (value && match) {