DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Added Fantasy Massage sites. Improved Private scraper, added movie link.

This commit is contained in:
ThePendulum 2020-02-08 04:52:32 +01:00
parent ff8ab2fe09
commit d2cb74a252
5 changed files with 143 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
migrations/20190325001339_releases.js
View File

@ -105,7 +105,7 @@ exports.up = knex => Promise.resolve()
table.string('name');
table.string('url');
table.text('description');
table.string('parameters');
table.text('parameters');
table.string('slug', 32)
.unique();
@ -140,7 +140,7 @@ exports.up = knex => Promise.resolve()
table.string('name');
table.string('url');
table.text('description');
table.string('parameters');
table.text('parameters');
table.integer('priority', 3)
.defaultTo(0);

60
seeds/01_sites.js
View File

@ -1457,14 +1457,26 @@ const sites = [
},
},
// FANTASY MASSAGE
{
slug: 'fantasymassage',
name: 'Fantasy Massage',
url: 'https://www.fantasymassage.com',
network: 'fantasymassage',
parameters: {
latest: 'https://www.fantasymassage.com/en/allvideos/fantasymassage/AllCategories/0/AllPornstars/0/updates/',
upcoming: 'https://www.fantasymassage.com/en/allvideos/fantasymassage/AllCategories/0/Actor/0/upcoming/',
},
},
{
slug: 'allgirlmassage',
name: 'All Girl Massage',
url: 'https://www.allgirlmassage.com',
network: 'fantasymassage',
parameters: {
latest: '/en/videos/updates/All-Categories/0/All-Pornstars/0/',
upcoming: '/en/videos/upcoming',
latest: 'https://www.fantasymassage.com/en/allvideos/allgirlmassage/AllCategories/0/AllPornstars/0/updates/',
upcoming: 'https://www.fantasymassage.com/en/allvideos/allgirlmassage/AllCategories/0/Actor/0/upcoming/',
deep: 'network',
photos: 'https://www.fantasymassage.com/en/photo/',
},
},
{
@ -1473,8 +1485,10 @@ const sites = [
url: 'https://www.nurumassage.com',
network: 'fantasymassage',
parameters: {
latest: '/en/videos/updates/All-Categories/0/All-Pornstars/0/',
upcoming: '/en/videos/upcoming',
latest: 'https://www.fantasymassage.com/en/allvideos/nurumassage/AllCategories/0/AllPornstars/0/updates/',
upcoming: 'https://www.fantasymassage.com/en/allvideos/nurumassage/AllCategories/0/Actor/0/upcoming/',
deep: 'network',
photos: 'https://www.fantasymassage.com/en/photo/',
},
},
{
@ -1483,9 +1497,10 @@ const sites = [
url: 'https://www.trickyspa.com',
network: 'fantasymassage',
parameters: {
latest: '/en/videos/updates/All-Categories/0/All-Pornstars/0/',
upcoming: '/en/videos/upcoming',
photos: '/en/photogallery/',
latest: 'https://www.fantasymassage.com/en/allvideos/trickyspa/AllCategories/0/AllPornstars/0/updates/',
upcoming: 'https://www.fantasymassage.com/en/allvideos/trickyspa/AllCategories/0/Actor/0/upcoming/',
deep: 'network',
photos: 'https://www.fantasymassage.com/en/photo/',
},
},
{
@ -1494,9 +1509,34 @@ const sites = [
url: 'https://www.soapymassage.com',
network: 'fantasymassage',
parameters: {
latest: '/en/videos/updates/All-Categories/0/All-Pornstars/0/',
upcoming: '/en/videos/upcoming',
photos: '/en/photogallery/',
latest: 'https://www.fantasymassage.com/en/allvideos/soapymassage/AllCategories/0/AllPornstars/0/updates/',
upcoming: 'https://www.fantasymassage.com/en/allvideos/soapymassage/AllCategories/0/Actor/0/upcoming/',
deep: 'network',
photos: 'https://www.fantasymassage.com/en/photo/',
},
},
{
slug: 'milkingtable',
name: 'Milking Table',
url: 'https://www.milkingtable.com',
network: 'fantasymassage',
parameters: {
latest: 'https://www.fantasymassage.com/en/allvideos/milkingtable/AllCategories/0/AllPornstars/0/updates/',
upcoming: 'https://www.fantasymassage.com/en/allvideos/milkingtable/AllCategories/0/Actor/0/upcoming/',
deep: 'network',
photos: 'https://www.fantasymassage.com/en/photo/',
},
},
{
slug: 'massageparlor',
name: 'Massage Parlor',
url: 'https://www.massage-parlor.com',
network: 'fantasymassage',
parameters: {
latest: 'https://www.fantasymassage.com/en/allvideos/massage-parlor/AllCategories/0/AllPornstars/0/updates/',
upcoming: 'https://www.fantasymassage.com/en/allvideos/massage-parlor/AllCategories/0/Actor/0/upcoming/',
deep: 'network',
photos: 'https://www.fantasymassage.com/en/photo/',
},
},
// GIRLSWAY

24
src/media.js
View File

@ -38,9 +38,15 @@ function pluckPhotos(photos, specifiedLimit) {
}
async function getEntropy(buffer) {
const { entropy } = await sharp(buffer).stats();
try {
const { entropy } = await sharp(buffer).stats();
return entropy;
return entropy;
} catch (error) {
logger.warn(`Failed to retrieve image entropy, using 7.5: ${error.message}`);
return 7.5;
}
}
async function createThumbnail(buffer) {
@ -80,7 +86,7 @@ function curatePhotoEntries(files) {
}));
}
async function findDuplicates(photos, identifier, prop = null, label) {
async function findDuplicates(photos, identifier, prop = null) {
const duplicates = await knex('media')
.whereIn(identifier, photos.flat().map((photo) => {
if (prop) return photo[prop];
@ -99,14 +105,6 @@ async function findDuplicates(photos, identifier, prop = null, label) {
return !duplicateLookup.has((prop && source[prop]) || (source.src && source));
});
if (duplicates.length > 0) {
logger.info(`${duplicates.length} media items already present by ${identifier} for ${label}`);
}
if (originals.length > 0) {
logger.info(`Fetching ${originals.length} new media items for ${label}`);
}
return [duplicates, originals];
}
@ -228,6 +226,8 @@ async function storePhotos(photos, {
const pluckedPhotos = pluckPhotos(Array.from(new Set(photos))); // pre-filter link duplicates, limit total per configuration
const [sourceDuplicates, sourceOriginals] = await findDuplicates(pluckedPhotos, 'source', null, label);
logger.info(`Fetching ${sourceOriginals.length} new ${role}s, ${sourceDuplicates.length} already present by source for ${label}`);
const metaFiles = await Promise.map(sourceOriginals, async (photoUrl, index) => fetchPhoto(photoUrl, index, label), {
concurrency: 10,
}).filter(photo => photo && photo.entropy > entropyFilter);
@ -235,6 +235,8 @@ async function storePhotos(photos, {
const metaFilesByHash = metaFiles.reduce((acc, photo) => ({ ...acc, [photo.hash]: photo }), {}); // pre-filter hash duplicates within set; may occur through fallbacks
const [hashDuplicates, hashOriginals] = await findDuplicates(Object.values(metaFilesByHash), 'hash', 'hash', label);
logger.info(`Saving ${hashOriginals.length} new ${role}s, ${hashDuplicates.length} already present by hash for ${label}`);
const savedPhotos = await savePhotos(hashOriginals, {
domain,
role,

61
src/scrapers/gamma.js
View File

@ -11,6 +11,16 @@ const logger = require('../logger')(__filename);
const { ex, get } = require('../utils/q');
const slugify = require('../utils/slugify');
function getAlbumUrl(albumPath, site) {
if (site.parameters?.photos) {
return /^http/.test(site.parameters.photos)
? `${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`
: `${site.url}${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`;
}
return `${site.url}${albumPath}`;
}
async function fetchPhotos(url) {
const res = await bhttp.get(url);
@ -25,7 +35,7 @@ function scrapePhotos(html) {
if (/\/join|\/createaccount/.test(url)) {
// URL links to join page instead of full photo, extract thumbnail
// /createaccount is used by e.g. Tricky Spa
// /createaccount is used by e.g. Tricky Spa native site
const src = $(linkEl).find('img').attr('src');
if (src.match('previews/')) {
@ -47,9 +57,9 @@ function scrapePhotos(html) {
}
async function getPhotos(albumPath, site) {
const albumUrl = site.parameters?.photos
? `${site.url}${site.parameters.photos}${albumPath.split('/').slice(-2).join('/')}`
: `${site.url}${albumPath}`;
const albumUrl = getAlbumUrl(albumPath, site);
console.log(albumUrl);
try {
const html = await fetchPhotos(albumUrl);
@ -397,28 +407,63 @@ async function fetchApiUpcoming(site) {
return fetchApiLatest(site, 1, true);
}
function getLatestUrl(site, page) {
if (site.parameters?.latest) {
return /^http/.test(site.parameters.latest)
? `${site.parameters.latest}${page}`
: `${site.url}${site.parameters.latest}${page}`;
}
return `${site.url}/en/videos/AllCategories/0/${page}`;
}
function getUpcomingUrl(site) {
if (site.parameters?.upcoming) {
return /^http/.test(site.parameters.upcoming)
? `${site.parameters.upcoming}`
: `${site.url}${site.parameters.upcoming}`;
}
return `${site.url}/en/videos/AllCategories/0/1/upcoming`;
}
async function fetchLatest(site, page = 1) {
const url = `${site.url}${site.parameters?.latest || '/en/videos/AllCategories/0/'}${page}`;
const url = getLatestUrl(site, page);
const res = await bhttp.get(url);
return scrapeAll(res.body.toString(), site);
}
async function fetchUpcoming(site) {
const url = `${site.url}${site.parameters?.upcoming || '/en/videos/AllCategories/0/1/upcoming'}`;
const url = getUpcomingUrl(site);
const res = await bhttp.get(url);
return scrapeAll(res.body.toString(), site);
}
function getDeepUrl(url, site) {
const { pathname } = new URL(url);
if (site.parameters?.deep === 'network') {
return `${site.network.url}${pathname}`;
}
if (site.parameters?.deep) {
return `${site.parameters.deep}${pathname}`;
}
return url;
}
async function fetchScene(url, site, release) {
if (site.parameters?.deep === false) {
return release;
}
const res = await bhttp.get(url);
const deepUrl = getDeepUrl(url, site);
const res = await bhttp.get(deepUrl);
return scrapeScene(res.body.toString(), url, site);
return scrapeScene(res.body.toString(), url, site, deepUrl);
}
async function fetchActorScenes(actorName, apiUrl, siteSlug) {

89
src/scrapers/private.js
View File

@ -5,9 +5,6 @@ const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
const fetchSites = require('../sites');
const { matchTags } = require('../tags');
async function getPhotos(entryId, site) {
const { hostname } = new URL(site.url);
@ -20,27 +17,6 @@ async function getPhotos(entryId, site) {
return photos;
}
async function getChannelSite($, site) {
const { hostname } = new URL(site.url);
if (site.isFallback && hostname.match('private.com')) {
const siteElement = $('.content-wrapper .logos-sites a');
const siteUrl = siteElement.attr('href').slice(0, -1);
const siteName = siteElement.text();
const siteSlug = siteName.replace(/\s+/g, '').toLowerCase();
const channelSite = await fetchSites({
slug: siteSlug,
name: siteName,
url: siteUrl,
});
return channelSite;
}
return site;
}
function scrapeLatest(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.content-wrapper .scene').toArray();
@ -82,55 +58,40 @@ function scrapeLatest(html, site) {
async function scrapeScene(html, url, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const release = {};
const entryId = url.split('/').slice(-1)[0];
const title = $('.video-wrapper meta[itemprop="name"]').attr('content');
[release.entryId] = url.split('/').slice(-1);
release.title = $('.video-wrapper meta[itemprop="name"]').attr('content');
release.description = $('.video-wrapper meta[itemprop="description"]').attr('content');
const date = moment.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate();
const actors = $('.content-wrapper .scene-models-list a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
release.date = moment.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate();
release.actors = $('.content-wrapper .scene-models-list a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
const description = $('.video-wrapper meta[itemprop="description"]').attr('content');
const [minutes, seconds] = $('.video-wrapper meta[itemprop="duration"]').attr('content').match(/\d+/g);
const duration = Number(minutes) * 60 + Number(seconds);
const timestamp = $('.video-wrapper meta[itemprop="duration"]').attr('content');
if (timestamp) {
const [minutes, seconds] = timestamp.match(/\d+/g);
release.duration = Number(minutes) * 60 + Number(seconds);
}
release.tags = $('.content-desc .scene-tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
release.likes = Number($('.content-desc #social-actions #likes').text());
const posterScript = $('script:contains(poster)').html();
const posterLink = posterScript.slice(posterScript.indexOf('https://'), posterScript.indexOf('.jpg') + 4);
const poster = $('meta[property="og:image"]').attr('content') || posterLink;
release.poster = $('meta[property="og:image"]').attr('content') || posterLink;
const trailerElementSrc = $('#videojs-trailer source').attr('src');
const trailer = $('meta[property="og:video"]').attr('content') || trailerElementSrc;
const likes = Number($('.content-desc #social-actions #likes').text());
const rawTags = $('.content-desc .scene-tags a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const [tags, photos, channelSite] = await Promise.all([
matchTags(rawTags),
getPhotos(entryId, site),
getChannelSite($, site),
]);
const scene = {
url,
entryId,
title,
date,
actors,
description,
duration,
tags,
poster,
photos,
trailer: {
src: trailer,
},
rating: {
likes,
},
site: channelSite || site,
release.trailer = {
src: $('meta[property="og:video"]').attr('content') || $('#videojs-trailer source').attr('src'),
};
return scene;
release.photos = await getPhotos(release.entryId, site);
release.movie = $('a[data-track="FULL MOVIE"]').attr('href');
const siteElement = $('.content-wrapper .logos-sites a');
if (siteElement) release.channel = siteElement.text().replace(/\s+/g, '').toLowerCase();
return release;
}
async function fetchLatest(site, page = 1) {