Allowing release scrapers to return actor details. Added True Amateurs.

This commit is contained in:
ThePendulum 2020-02-09 23:25:54 +01:00
parent 739d2de297
commit 139f0ce7cb
62 changed files with 47 additions and 20 deletions

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.2 KiB

After

Width:  |  Height:  |  Size: 2.9 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 21 KiB

After

Width:  |  Height:  |  Size: 22 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 34 KiB

After

Width:  |  Height:  |  Size: 36 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.8 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 7.6 KiB

After

Width:  |  Height:  |  Size: 5.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 108 KiB

After

Width:  |  Height:  |  Size: 110 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 21 KiB

After

Width:  |  Height:  |  Size: 21 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 49 KiB

After

Width:  |  Height:  |  Size: 49 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 101 KiB

After

Width:  |  Height:  |  Size: 101 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 90 KiB

After

Width:  |  Height:  |  Size: 90 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 14 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 17 KiB

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.6 KiB

After

Width:  |  Height:  |  Size: 5.6 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 17 KiB

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 41 KiB

After

Width:  |  Height:  |  Size: 41 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 42 KiB

After

Width:  |  Height:  |  Size: 42 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 20 KiB

After

Width:  |  Height:  |  Size: 20 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 51 KiB

After

Width:  |  Height:  |  Size: 52 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 38 KiB

After

Width:  |  Height:  |  Size: 40 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 105 KiB

After

Width:  |  Height:  |  Size: 105 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 61 KiB

After

Width:  |  Height:  |  Size: 61 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 39 KiB

After

Width:  |  Height:  |  Size: 39 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 941 B

After

Width:  |  Height:  |  Size: 1.6 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 61 KiB

After

Width:  |  Height:  |  Size: 61 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 64 KiB

After

Width:  |  Height:  |  Size: 65 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 7.6 KiB

After

Width:  |  Height:  |  Size: 7.6 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 17 KiB

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 10 KiB

After

Width:  |  Height:  |  Size: 10 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 45 KiB

After

Width:  |  Height:  |  Size: 46 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 95 KiB

After

Width:  |  Height:  |  Size: 95 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 45 KiB

After

Width:  |  Height:  |  Size: 47 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 9.1 KiB

After

Width:  |  Height:  |  Size: 9.1 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 18 KiB

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 108 KiB

After

Width:  |  Height:  |  Size: 110 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 51 KiB

After

Width:  |  Height:  |  Size: 51 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 104 KiB

After

Width:  |  Height:  |  Size: 104 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 34 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 59 KiB

After

Width:  |  Height:  |  Size: 60 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 240 KiB

After

Width:  |  Height:  |  Size: 240 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 125 KiB

After

Width:  |  Height:  |  Size: 125 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 34 KiB

After

Width:  |  Height:  |  Size: 17 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 45 KiB

After

Width:  |  Height:  |  Size: 45 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 52 KiB

After

Width:  |  Height:  |  Size: 52 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 82 KiB

After

Width:  |  Height:  |  Size: 84 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 25 KiB

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 9.2 KiB

After

Width:  |  Height:  |  Size: 9.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 139 KiB

After

Width:  |  Height:  |  Size: 139 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 24 KiB

After

Width:  |  Height:  |  Size: 24 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 85 KiB

After

Width:  |  Height:  |  Size: 85 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 97 KiB

After

Width:  |  Height:  |  Size: 97 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 36 KiB

After

Width:  |  Height:  |  Size: 38 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 38 KiB

After

Width:  |  Height:  |  Size: 38 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 26 KiB

After

Width:  |  Height:  |  Size: 27 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 24 KiB

After

Width:  |  Height:  |  Size: 24 KiB

View File

@ -2097,6 +2097,14 @@ const sites = [
parameters: { native: true }, parameters: { native: true },
network: 'mindgeek', network: 'mindgeek',
}, },
{
slug: 'trueamateurs',
name: 'True Amateurs',
url: 'https://www.trueamateurs.com',
description: 'TrueAmateurs.com is the best homemade porn from real amateurs. Watch these real hot couples in our exclusive scenes.',
parameters: { native: true },
network: 'mindgeek',
},
// MIKE ADRIANO // MIKE ADRIANO
{ {
slug: 'trueanal', slug: 'trueanal',

View File

@ -457,22 +457,20 @@ async function scrapeBasicActors() {
} }
async function associateActors(mappedActors, releases) { async function associateActors(mappedActors, releases) {
const actorMap = Object.keys(mappedActors).reduce((acc, actorName) => ({ ...acc, [actorName]: slugify(actorName) }), {});
const [existingActorEntries, existingAssociationEntries] = await Promise.all([ const [existingActorEntries, existingAssociationEntries] = await Promise.all([
knex('actors') knex('actors')
.whereIn('name', Object.keys(actorMap)) .whereIn('name', Object.values(mappedActors).map(actor => actor.name))
.orWhereIn('slug', Object.values(actorMap)), .orWhereIn('slug', Object.keys(mappedActors)),
knex('releases_actors').whereIn('release_id', releases.map(release => release.id)), knex('releases_actors').whereIn('release_id', releases.map(release => release.id)),
]); ]);
const associations = await Promise.map(Object.entries(mappedActors), async ([actorName, releaseIds]) => { const associations = await Promise.map(Object.entries(mappedActors), async ([actorSlug, actor]) => {
try { try {
const actorEntry = existingActorEntries.find(actor => actor.slug === actorMap[actorName]) const actorEntry = existingActorEntries.find(actorX => actorX.slug === actorSlug)
|| await storeActor({ name: actorName }); || await storeActor(actor);
// if a scene // if a scene
return Array.from(releaseIds) return Array.from(actor.releaseIds)
.map(releaseId => ({ .map(releaseId => ({
release_id: releaseId, release_id: releaseId,
actor_id: actorEntry.id, actor_id: actorEntry.id,
@ -482,7 +480,7 @@ async function associateActors(mappedActors, releases) {
.some(associationEntry => associationEntry.actor_id === association.actor_id .some(associationEntry => associationEntry.actor_id === association.actor_id
&& associationEntry.release_id === association.release_id)); && associationEntry.release_id === association.release_id));
} catch (error) { } catch (error) {
logger.error(actorName, error); logger.error(actor.name, error);
return null; return null;
} }
}); });

View File

@ -289,9 +289,18 @@ function accumulateActors(releases) {
release.actors.forEach((actor) => { release.actors.forEach((actor) => {
const actorName = actor.name ? actor.name.trim() : actor.trim(); const actorName = actor.name ? actor.name.trim() : actor.trim();
const actorSlug = slugify(actorName);
if (!acc[actorName]) acc[actorName] = new Set(); if (!acc[actorSlug]) {
acc[actorName].add(release.id); acc[actorSlug] = {
name: actorName,
slug: actorSlug,
releaseIds: new Set(),
};
}
if (actor.name) acc[actorSlug] = { ...acc[actorSlug], ...actor }; // actor input contains profile info
acc[actorSlug].releaseIds.add(release.id);
}); });
return acc; return acc;

View File

@ -7,6 +7,11 @@ const slugify = require('../utils/slugify');
const clusterId = '617fb597b659459bafe6472470d9073a'; const clusterId = '617fb597b659459bafe6472470d9073a';
const authKey = 'YmFuZy1yZWFkOktqVDN0RzJacmQ1TFNRazI='; const authKey = 'YmFuZy1yZWFkOktqVDN0RzJacmQ1TFNRazI=';
const genderMap = {
M: 'male',
F: 'female',
};
function getScreenUrl(item, scene) { function getScreenUrl(item, scene) {
return `https://i.bang.com/screenshots/${scene.dvd.id}/movie/${scene.order}/${item.screenId}.jpg`; return `https://i.bang.com/screenshots/${scene.dvd.id}/movie/${scene.order}/${item.screenId}.jpg`;
} }
@ -37,7 +42,6 @@ function scrapeScene(scene, site) {
entryId: scene.id, entryId: scene.id,
title: scene.name, title: scene.name,
description: scene.description, description: scene.description,
actors: scene.actors.map(actor => actor.name),
tags: scene.genres.concat(scene.actions).map(genre => genre.name), tags: scene.genres.concat(scene.actions).map(genre => genre.name),
duration: scene.duration, duration: scene.duration,
}; };
@ -48,6 +52,8 @@ function scrapeScene(scene, site) {
const date = new Date(scene.releaseDate); const date = new Date(scene.releaseDate);
release.date = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate())); release.date = new Date(Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate()));
release.actors = scene.actors.map(actor => ({ name: actor.name, gender: genderMap[actor.gender] }));
if (scene.is4k) release.tags.push('4k'); if (scene.is4k) release.tags.push('4k');
if (scene.gay) release.tags.push('gay'); if (scene.gay) release.tags.push('gay');

View File

@ -106,7 +106,7 @@ async function scrapeApiReleases(json, site) {
else if (site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`; else if (site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`;
release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate(); release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate();
release.actors = scene.actors.map(({ name }) => name); release.actors = scene.actors.map(actor => ({ name: actor.name, gender: actor.gender }));
release.director = scene.directors[0]?.name || null; release.director = scene.directors[0]?.name || null;
release.tags = scene.master_categories release.tags = scene.master_categories
@ -198,15 +198,21 @@ async function scrapeScene(html, url, site, scrapedRelease) {
if (data.director?.[0]?.name) release.director = data.director[0].name; if (data.director?.[0]?.name) release.director = data.director[0].name;
else if (data2?.director?.[0]?.name) release.director = data2.director[0].name; else if (data2?.director?.[0]?.name) release.director = data2.director[0].name;
const actors = data?.actor || data2?.actor || [];
release.actors = actors.map(actor => actor.name);
const stars = (data.aggregateRating.ratingValue / data.aggregateRating.bestRating) * 5; const stars = (data.aggregateRating.ratingValue / data.aggregateRating.bestRating) * 5;
if (stars) release.rating = { stars }; if (stars) release.rating = { stars };
release.duration = moment.duration(data.duration.slice(2)).asSeconds(); release.duration = moment.duration(data.duration.slice(2)).asSeconds();
} }
const actors = data?.actor || data2?.actor;
if (actors) {
release.actors = actors.map(actor => ({
name: actor.name,
gender: actor.gender,
}));
}
const hasTrans = release.actors.some(actor => actor.gender === 'shemale'); const hasTrans = release.actors.some(actor => actor.gender === 'shemale');
const rawTags = data?.keywords?.split(', ') || data2?.keywords?.split(', '); const rawTags = data?.keywords?.split(', ') || data2?.keywords?.split(', ');
release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags; release.tags = hasTrans ? [...rawTags, 'transsexual'] : rawTags;
@ -440,7 +446,7 @@ async function fetchUpcoming(site) {
} }
function getDeepUrl(url, site, release) { function getDeepUrl(url, site, release) {
const pathname = release.path || new URL(url).pathname; const pathname = release?.path || new URL(url).pathname;
if (site.parameters?.deep === 'network') { if (site.parameters?.deep === 'network') {
return `${site.network.url}${pathname}`; return `${site.network.url}${pathname}`;

View File

@ -40,7 +40,7 @@ function scrapeLatestX(data, site) {
const hostname = site.parameters?.native ? site.url : site.network.url; const hostname = site.parameters?.native ? site.url : site.network.url;
const url = `${hostname}/scene/${entryId}/`; const url = `${hostname}/scene/${entryId}/`;
const date = new Date(data.dateReleased); const date = new Date(data.dateReleased);
const actors = data.actors.map(actor => actor.name); const actors = data.actors.map(actor => ({ name: actor.name, gender: actor.gender }));
const tags = data.tags.map(tag => tag.name); const tags = data.tags.map(tag => tag.name);
@ -83,7 +83,7 @@ function scrapeScene(data, url, _site, networkName) {
release.description = description; release.description = description;
release.date = new Date(data.dateReleased); release.date = new Date(data.dateReleased);
release.actors = data.actors.map(actor => actor.name); release.actors = data.actors.map(actor => ({ name: actor.name, gender: actor.gender }));
release.tags = data.tags.map(tag => tag.name); release.tags = data.tags.map(tag => tag.name);

View File

@ -34,7 +34,7 @@ async function fetchClassic(site, page) {
} }
async function fetchLatestWrap(site, page = 1) { async function fetchLatestWrap(site, page = 1) {
if (site.parameters && site.parameters.classic) { if (site.parameters?.classic) {
return fetchClassic(site, page); return fetchClassic(site, page);
} }