6 Commits

Author SHA1 Message Date
DebaucheryLibrarian
661b8b716b 1.210.1 2022-03-09 23:26:50 +01:00
DebaucheryLibrarian
5ff076cac3 Added DP Star Sex Challenges to Digital Playground. 2022-03-09 23:26:48 +01:00
DebaucheryLibrarian
41c100ac4e 1.210.0 2022-03-04 23:32:28 +01:00
DebaucheryLibrarian
c6e977f842 Added movie support to MindGeek scraper. 2022-03-04 23:32:09 +01:00
DebaucheryLibrarian
50b7f521b5 1.209.5 2022-02-27 21:15:07 +01:00
DebaucheryLibrarian
f0d0ee3acc Removed redundant scenes path from Digital Playground main channel. 2022-02-27 21:15:05 +01:00
23 changed files with 325 additions and 87 deletions

View File

@@ -108,6 +108,7 @@
:fetch-releases="fetchEntity"
:items-total="totalCount"
:items-per-page="limit"
:available-tags="entity.tags"
/>
<div class="releases">

View File

@@ -102,6 +102,8 @@ export default {
}
.name {
display: flex;
align-items: center;
color: var(--text-light);
font-size: 1.25rem;
font-weight: bold;

View File

@@ -11,16 +11,31 @@
class="empty"
>No results for "{{ $route.query.query }}"</span>
<div
v-else
class="entity-tiles"
>
<Entity
v-for="entity in entities"
:key="entity.parent ? `entity-tile-${entity.parent.slug}-${entity.slug}` : `entity-tile-${entity.slug}`"
:entity="entity"
/>
</div>
<template v-else>
<h2 class="heading">Popular</h2>
<div
class="entity-tiles"
>
<Entity
v-for="entity in popularEntities"
:key="entity.parent ? `entity-tile-${entity.parent.slug}-${entity.slug}` : `entity-tile-${entity.slug}`"
:entity="entity"
/>
</div>
<h2 class="heading">All networks</h2>
<div
class="entity-tiles"
>
<Entity
v-for="entity in entities"
:key="entity.parent ? `entity-tile-${entity.parent.slug}-${entity.slug}` : `entity-tile-${entity.slug}`"
:entity="entity"
/>
</div>
</template>
</div>
<Footer />
@@ -58,6 +73,45 @@ async function searchEntities() {
this.done = true;
}
function popularEntities() {
const entitiesBySlug = Object.fromEntries(this.entities.map((entity) => [entity.slug, entity]));
return [
'21sextury',
'amateurallure',
'analvids',
'bamvisions',
'bang',
'bangbros',
'blowpass',
'brazzers',
'burningangel',
'digitalplayground',
'dogfartnetwork',
'dorcel',
'elegantangel',
'evilangel',
'fakehub',
'girlsway',
'hookuphotshot',
'hussiepass',
'insex',
'julesjordan',
'kellymadison',
'kink',
'mofos',
'naughtyamerica',
'newsensations',
'pervcity',
'pornpros',
'private',
'realitykings',
'twistys',
'vixen',
'xempire',
].map((slug) => entitiesBySlug[slug]).filter(Boolean);
}
async function mounted() {
this.pageTitle = 'Channels';
@@ -82,6 +136,7 @@ export default {
},
computed: {
channelCount,
popularEntities,
},
watch: {
$route: fetchEntities,
@@ -130,6 +185,10 @@ export default {
font-weight: bold;
}
.heading {
margin: 1rem 0 0 0;
}
@media(max-width: $breakpoint2) {
.entity-tiles {
grid-gap: .5rem;

View File

@@ -3,12 +3,6 @@
<div class="content-inner">
<SearchBar :placeholder="`Search ${totalCount} movies`" />
<TagFilter
class="filters-filter"
:filter="filter"
:available-tags="availableTags"
/>
<div
ref="tiles"
class="tiles"
@@ -36,7 +30,6 @@
import MovieTile from './movie-tile.vue';
import SearchBar from '../search/bar.vue';
import Pagination from '../pagination/pagination.vue';
import TagFilter from '../filters/tag-filter.vue';
async function fetchMovies() {
if (this.$route.query.query) {
@@ -80,7 +73,6 @@ export default {
MovieTile,
SearchBar,
Pagination,
TagFilter,
},
data() {
return {

View File

@@ -87,7 +87,6 @@ const tagSlugsByCategory = {
'titty-fucking',
'fisting',
'anal-fisting',
'fisting-dp',
],
group: [
'mfm',
@@ -108,16 +107,6 @@ const tagSlugsByCategory = {
'bukkake',
'fake-cum',
],
toys: [
'toys',
'toy-anal',
'toy-dp',
'double-dildo',
'double-dildo-blowjob',
'double-dildo-kiss',
'double-dildo-anal',
'double-dildo-dp',
],
roleplay: [
'family',
'parody',
@@ -126,6 +115,15 @@ const tagSlugsByCategory = {
'maid',
'nun',
],
extreme: [
'dp',
'airtight',
'dap',
'dvp',
'triple-penetration',
'tap',
'tvp',
],
fetish: [
'bdsm',
'femdom',
@@ -134,15 +132,15 @@ const tagSlugsByCategory = {
'latex',
'blindfold',
],
extreme: [
'dp',
'airtight',
'dap',
'dvp',
'da-tp',
'dv-tp',
'tap',
'tvp',
toys: [
'toys',
'toy-anal',
'toy-dp',
'double-dildo',
'double-dildo-blowjob',
'double-dildo-kiss',
'double-dildo-anal',
'double-dildo-dp',
],
misc: [
'gaping',

View File

@@ -105,6 +105,7 @@ function curateEntity(entity, parent, releases) {
};
if (entity.tags) curatedEntity.tags = entity.tags.map(({ tag }) => tag);
if (entity.sceneTags) curatedEntity.sceneTags = entity.sceneTags;
if (entity.children) {
if (entity.children.nodes) {

View File

@@ -41,6 +41,11 @@ function initEntitiesActions(store, router) {
slug
}
}
sceneTags {
id
name
slug
}
children: childEntitiesConnection(
orderBy: [PRIORITY_DESC, NAME_ASC],
filter: {

View File

@@ -1,5 +1,5 @@
exports.up = async (knex) => knex.raw(`
CREATE OR REPLACE FUNCTION entities_scene_total(entity entities) RETURNS integer AS $$
CREATE OR REPLACE FUNCTION entities_scene_total(entity entities) RETURNS bigint AS $$
SELECT COUNT(id)
FROM releases
WHERE releases.entity_id = entity.id;

View File

@@ -1,8 +0,0 @@
exports.up = async (knex) => knex.raw(`
CREATE VIEW movies_tagged AS
SELECT * FROM movies;
`);
exports.down = async (knex) => knex.raw(`
DROP VIEW IF EXISTS movies_tagged;
`);

View File

@@ -0,0 +1,23 @@
exports.up = async (knex) => knex.raw(`
CREATE FUNCTION entities_scene_tags(entity entities, selectable_tags text[]) RETURNS SETOF tags AS $$
SELECT tags.*
FROM releases
LEFT JOIN
releases_tags ON releases_tags.release_id = releases.id
LEFT JOIN
tags ON tags.id = releases_tags.tag_id
WHERE
releases.entity_id = entity.id
AND
CASE WHEN array_length(selectable_tags, 1) IS NOT NULL
THEN tags.slug = ANY(selectable_tags)
ELSE true
END
GROUP BY tags.id
ORDER BY tags.name;
$$ LANGUAGE SQL STABLE;
`);
exports.down = async (knex) => knex.raw(`
DROP FUNCTION IF EXISTS entities_tags;
`);

View File

@@ -0,0 +1,98 @@
exports.up = async (knex) => Promise.resolve()
.then(() => knex.schema.createTable('series', (table) => {
table.increments('id', 16);
table.integer('entity_id', 12)
.references('id')
.inTable('entities')
.notNullable();
table.integer('studio_id', 12)
.references('id')
.inTable('entities');
table.text('entry_id');
table.unique(['entity_id', 'entry_id']);
table.text('url', 1000);
table.text('title');
table.text('slug');
table.timestamp('date');
table.index('date');
table.enum('date_precision', ['year', 'month', 'week', 'day', 'hour', 'minute', 'second'])
.defaultTo('day');
table.text('description');
table.boolean('deep');
table.text('deep_url', 1000);
table.text('comment');
table.integer('created_batch_id', 12)
.references('id')
.inTable('batches')
.onDelete('cascade');
table.integer('updated_batch_id', 12)
.references('id')
.inTable('batches')
.onDelete('cascade');
table.datetime('created_at')
.defaultTo(knex.fn.now());
}))
.then(() => knex.schema.createTable('series_scenes', (table) => {
table.integer('serie_id', 16)
.notNullable()
.references('id')
.inTable('series')
.onDelete('cascade');
table.integer('scene_id', 16)
.notNullable()
.references('id')
.inTable('releases')
.onDelete('cascade');
table.unique(['serie_id', 'scene_id']);
table.datetime('created_at')
.defaultTo(knex.fn.now());
}))
.then(() => knex.schema.createTable('series_trailers', (table) => {
table.integer('serie_id', 16)
.unique()
.notNullable()
.references('id')
.inTable('series')
.onDelete('cascade');
table.text('media_id', 21)
.notNullable()
.references('id')
.inTable('media');
}))
.then(() => knex.schema.createTable('series_posters', (table) => {
table.integer('serie_id', 16)
.notNullable()
.references('id')
.inTable('series')
.onDelete('cascade');
table.text('media_id', 21)
.notNullable()
.references('id')
.inTable('media')
.onDelete('cascade');
table.unique('serie_id');
}));
exports.down = async (knex) => Promise.resolve()
.then(() => knex.schema.dropTable('series_covers'))
.then(() => knex.schema.dropTable('series_trailers'))
.then(() => knex.schema.dropTable('series_posters'))
.then(() => knex.schema.dropTable('series'));

4
package-lock.json generated
View File

@@ -1,12 +1,12 @@
{
"name": "traxxx",
"version": "1.209.4",
"version": "1.210.1",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "traxxx",
"version": "1.209.4",
"version": "1.210.1",
"license": "ISC",
"dependencies": {
"@casl/ability": "^5.2.2",

View File

@@ -1,6 +1,6 @@
{
"name": "traxxx",
"version": "1.209.4",
"version": "1.210.1",
"description": "All the latest porn releases in one place",
"main": "src/app.js",
"scripts": {

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.3 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 29 KiB

View File

@@ -2646,7 +2646,7 @@ const sites = [
{
slug: 'digitalplayground',
name: 'Digital Playground',
url: 'https://www.digitalplayground.com/scenes',
url: 'https://www.digitalplayground.com',
description: '',
parameters: { extract: true },
parent: 'digitalplayground',
@@ -2680,6 +2680,13 @@ const sites = [
description: '',
parent: 'digitalplayground',
},
{
slug: 'dpstarsexchallenges',
name: 'DP Star Sex Challenges',
url: 'https://www.digitalplayground.com/scenes?site=210',
parent: 'digitalplayground',
hasLogo: false,
},
{
slug: 'blockbuster',
name: 'Blockbuster',

View File

@@ -727,8 +727,6 @@ const tagMedia = [
['da-tp', 7, 'Polly Petrova in YE069', 'analvids'],
['da-tp', 5, 'Venera Maxima in GIO1287'],
['da-tp', 6, 'Adriana Chechik in "Gangbang Me"', 'hardx'],
['da-tp', 0, 'Natasha Teen in SZ2164'],
['da-tp', 1, 'Francys Belle in SZ1702', 'analvids'],
['dap', 7, 'Adriana Chechik in "DP Masters 6"', 'julesjordan'],
['dap', 10, 'Kira Noir', 'hardx'],
['dap', 'emily_pink_legalporno', 'Emily Pink', 'analvids'],
@@ -1049,6 +1047,7 @@ const tagMedia = [
['toy-dp', 0, 'Marley Brinx, Ivy Lebelle and Lyra Law in "Marley Brinx First GGDP"', 'lesbianx'],
['toys', 1, 'Chloe Lamour in "Curives In All The Right Places"', 'wetandpuffy'],
['toys', 'shawna_lenee_sunrisekings', 'Shawna Lenee', 'sunrisekings'],
['triple-penetration', 'lucky_bee_analvids', 'Lucky Bee', 'analvids'],
['triple-penetration', 'angela_white_julesjordan', 'Angela White in "Her Biggest Gangbang Ever"', 'julesjordan'],
['triple-penetration', 'ria_sunn_legalporno', 'Ria Sunn in SZ2082', 'analvids'],
['tvp', 'september_reign_wefuckblackgirls', 'September Reign in "Second Appearance"', 'wefuckblackgirls'],

View File

@@ -20,6 +20,7 @@ const scrapers = require('./scrapers/scrapers').actors;
const argv = require('./argv');
const include = require('./utils/argv-include')(argv);
const bulkInsert = require('./utils/bulk-insert');
const chunk = require('./utils/chunk');
const logger = require('./logger')(__filename);
const { toBaseReleases } = require('./deep');
@@ -1048,33 +1049,42 @@ async function flushProfiles(actorIdsOrNames) {
logger.info(`Removed ${deleteCount} profiles`);
}
async function deleteActors(actorIdsOrNames) {
const actors = await knex('actors')
.whereIn('id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number'))
.orWhere((builder) => {
builder
.whereIn('name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string'))
.whereNull('entity_id');
});
async function deleteActors(allActorIdsOrNames) {
const deleteCounts = await Promise.map(chunk(allActorIdsOrNames), async (actorIdsOrNames) => {
const actors = await knex('actors')
.whereIn('id', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'number'))
.orWhere((builder) => {
builder
.whereIn('name', actorIdsOrNames.filter((idOrName) => typeof idOrName === 'string'))
.whereNull('entity_id');
});
const actorIds = actors.map((actor) => actor.id);
const actorIds = actors.map((actor) => actor.id);
const sceneIds = await knex('releases_actors')
.select('releases.id')
.whereIn('actor_id', actorIds)
.leftJoin('releases', 'releases.id', 'releases_actors.release_id')
.pluck('id');
const sceneIds = await knex('releases_actors')
.select('releases.id')
.whereIn('actor_id', actorIds)
.leftJoin('releases', 'releases.id', 'releases_actors.release_id')
.pluck('id');
const [deletedScenesCount, deletedActorsCount] = await Promise.all([
deleteScenes(sceneIds),
knex('actors')
.whereIn('id', actorIds)
.delete(),
]);
const [deletedScenesCount, deletedActorsCount] = await Promise.all([
deleteScenes(sceneIds),
knex('actors')
.whereIn('id', actorIds)
.delete(),
]);
return { deletedScenesCount, deletedActorsCount };
}, { concurrency: 10 });
const deletedActorsCount = deleteCounts.reduce((acc, count) => acc + count.deletedActorsCount, 0);
const deletedScenesCount = deleteCounts.reduce((acc, count) => acc + count.deletedScenesCount, 0);
await flushOrphanedMedia();
logger.info(`Removed ${deletedActorsCount} actors with ${deletedScenesCount} scenes`);
return deletedActorsCount;
}
async function flushActors() {

View File

@@ -961,9 +961,12 @@ async function flushOrphanedMedia() {
await deleteS3Objects(orphanedMedia.filter((media) => media.is_s3));
}
await fsPromises.rm(path.join(config.media.path, 'temp'), { recursive: true });
logger.info('Cleared temporary media directory');
try {
await fsPromises.rm(path.join(config.media.path, 'temp'), { recursive: true });
logger.info('Cleared temporary media directory');
} catch (error) {
logger.warn(`Failed to clear temporary media directory: ${error.message}`);
}
}
module.exports = {

View File

@@ -11,6 +11,12 @@ const slugify = require('../utils/slugify');
const http = require('../utils/http');
const { inchesToCm, lbsToKg } = require('../utils/convert');
function getBasePath(channel, path = '/scene') {
return channel.parameters?.scene
|| ((channel.parameters?.native || channel.type === 'network') && `${channel.url}${path}`)
|| `${channel.parent.url}${path}`;
}
function getThumbs(scene) {
if (scene.images.poster) {
return Object.values(scene.images.poster) // can be { 0: {}, 1: {}, ... } instead of array
@@ -18,7 +24,7 @@ function getThumbs(scene) {
.map((image) => image.xl.url);
}
if (scene.images.card_main_rect) {
if (Array.isArray(scene.images.card_main_rect)) {
return scene.images.card_main_rect
.concat(scene.images.card_secondary_rect || [])
.map((image) => image.xl.url.replace('.thumb', ''));
@@ -27,6 +33,20 @@ function getThumbs(scene) {
return [];
}
function getCovers(images) {
return [
[
images.cover[0].md?.url,
images.cover[0].sm?.url,
images.cover[0].xs?.url,
// bigger but usually upscaled
images.cover[0].xx?.url,
images.cover[0].xl?.url,
images.cover[0].lg?.url,
],
];
}
function getVideos(data) {
const teaserSources = data.videos.mediabook?.files;
const trailerSources = data.children.find((child) => child.type === 'trailer')?.videos.full?.files;
@@ -51,9 +71,7 @@ function scrapeLatestX(data, site, filterChannel) {
description: data.description,
};
const basepath = site.parameters?.scene
|| (site.parameters?.native && `${site.url}/scene`)
|| `${site.parent.url}/scene`;
const basepath = getBasePath(site);
release.url = `${basepath}/${release.entryId}/${slugify(release.title)}`;
release.date = new Date(data.dateReleased);
@@ -84,6 +102,9 @@ function scrapeLatestX(data, site, filterChannel) {
};
}
const siteName = data.collections[0]?.name || data.brand;
release.channel = slugify(siteName, '');
return release;
}
@@ -96,7 +117,7 @@ async function scrapeLatest(items, site, filterChannel) {
};
}
function scrapeScene(data, url, _site, networkName) {
function scrapeRelease(data, url, channel, networkName) {
const release = {};
const { id: entryId, title, description } = data;
@@ -129,6 +150,29 @@ function scrapeScene(data, url, _site, networkName) {
release.url = url || `https://www.${networkName || data.brand}.com/scene/${entryId}/`;
if (data.parent?.type === 'movie' || data.parent?.type === 'serie') {
release[data.parent.type] = {
entryId: data.parent.id,
url: `${getBasePath(channel, '/movie')}/${data.parent.id}/${slugify(data.parent.title, '-', { removePunctuation: true })}`,
title: data.parent.title,
description: data.parent.description,
date: new Date(data.parent.dateReleased),
channel: slugify(data.parent.collections?.name || data.parent.brand),
covers: getCovers(data.parent.images),
shallow: true,
};
}
if (data.type === 'movie') {
release.covers = getCovers(data.images);
release.scenes = data.children?.map((scene) => ({
entryId: scene.id,
url: `${getBasePath(channel)}/${scene.id}/${slugify(scene.title)}`,
title: scene.title,
shallow: true,
}));
}
return release;
}
@@ -230,7 +274,7 @@ function scrapeProfile(data, html, releases = [], networkName) {
profile.naturalBoobs = false;
}
profile.releases = releases.map((release) => scrapeScene(release, null, null, networkName));
profile.releases = releases.map((release) => scrapeRelease(release, null, null, networkName));
return profile;
}
@@ -292,8 +336,8 @@ async function fetchUpcoming(site, page, options) {
return res.statusCode;
}
async function fetchScene(url, site, baseScene, options) {
if (baseScene?.entryId) {
async function fetchRelease(url, site, baseScene, options) {
if (baseScene?.entryId && !baseScene.shallow) {
// overview and deep data is the same, don't hit server unnecessarily
return baseScene;
}
@@ -312,7 +356,7 @@ async function fetchScene(url, site, baseScene, options) {
if (res.status === 200 && res.body.result) {
return {
scene: scrapeScene(res.body.result, url, site),
scene: scrapeRelease(res.body.result, url, site),
};
}
@@ -374,6 +418,7 @@ module.exports = {
scrapeLatestX,
fetchLatest,
fetchUpcoming,
fetchScene,
fetchScene: fetchRelease,
fetchMovie: fetchRelease,
fetchProfile,
};

View File

@@ -142,6 +142,7 @@ async function getTrailer(scene, channel, url) {
return null;
}
/*
async function getPhotosLegacy(url) {
const htmlRes = await http.get(url, {
extract: {
@@ -169,6 +170,7 @@ async function getPhotosLegacy(url) {
return [];
}
}
*/
async function getPhotos(url) {
const htmlRes = await http.get(url, {

View File

@@ -392,7 +392,8 @@ async function associateMovieScenes(movies, movieScenes) {
return null;
}
const sceneMovie = moviesByEntityIdAndEntryId[scene.entity.id]?.[scene.movie.entryId];
const sceneMovie = moviesByEntityIdAndEntryId[scene.entity.id]?.[scene.movie.entryId]
|| moviesByEntityIdAndEntryId[scene.entity.parent?.id]?.[scene.movie.entryId];
if (sceneMovie?.id) {
return {