Added screen caps separate from photos. Added Tokyo Hot. Added hair type, shoe size and blood type actor fields.

This commit is contained in:
DebaucheryLibrarian 2023-07-25 03:03:41 +02:00
parent 6fe212796b
commit 693983dc29
32 changed files with 472 additions and 113 deletions

View File

@ -2,7 +2,7 @@
<div class="media-container"> <div class="media-container">
<div <div
class="media" class="media"
:class="{ center: (release.photos?.length || 0) + (release.scenesPhotos?.length || 0) < 2, preview: !me }" :class="{ center: (release.photos?.length || 0) + (release.caps?.length || 0) + (release.scenesPhotos?.length || 0) < 2, preview: !me }"
> >
<div <div
v-if="release.trailer || release.teaser" v-if="release.trailer || release.teaser"
@ -169,7 +169,7 @@ function photos() {
const clips = this.release.clips || []; const clips = this.release.clips || [];
const clipPostersById = clips.reduce((acc, clip) => ({ ...acc, [clip.poster.id]: clip.poster }), {}); const clipPostersById = clips.reduce((acc, clip) => ({ ...acc, [clip.poster.id]: clip.poster }), {});
const uniqueClipPosters = Array.from(new Set(clips.map((clip) => clip.poster.id) || [])).map((posterId) => clipPostersById[posterId]); const uniqueClipPosters = Array.from(new Set(clips.map((clip) => clip.poster.id) || [])).map((posterId) => clipPostersById[posterId]);
const photosWithClipPosters = (this.release.photos || []).concat(this.release.scenesPhotos || []).concat(uniqueClipPosters); const photosWithClipPosters = (this.release.photos || []).concat(this.release.caps || []).concat(this.release.scenesPhotos || []).concat(uniqueClipPosters);
if (this.release.trailer || (this.release.teaser && this.release.teaser.mime !== 'image/gif')) { if (this.release.trailer || (this.release.teaser && this.release.teaser.mime !== 'image/gif')) {
// poster will be on trailer video // poster will be on trailer video

View File

@ -21,14 +21,14 @@
<Details :release="release" /> <Details :release="release" />
<button <button
v-if="release.photos?.length > 0 || release.scenesPhotos?.length > 0" v-if="showAlbum"
class="album-toggle" class="album-toggle"
@click="$router.push({ hash: '#album' })" @click="$router.push({ hash: '#album' })"
><Icon icon="grid3" />View album</button> ><Icon icon="grid3" />View album</button>
<Album <Album
v-if="showAlbum" v-if="showAlbum && $route.hash === '#album'"
:items="[release.poster, ...(release.photos || []), ...(release.scenesPhotos || [])]" :items="[release.poster, ...(release.photos || []), ...(release.caps || []), ...(release.scenesPhotos || [])]"
:title="release.title" :title="release.title"
:path="config.media.mediaPath" :path="config.media.mediaPath"
@close="$router.replace({ hash: undefined })" @close="$router.replace({ hash: undefined })"
@ -391,7 +391,7 @@ function pageTitle() {
} }
function showAlbum() { function showAlbum() {
return (this.release.photos?.length > 0 || this.release.scenesPhotos?.length > 0) && this.$route.hash === '#album'; return this.release.photos?.length > 0 || this.release.caps?.length > 0 || this.release.scenesPhotos?.length > 0;
} }
async function mounted() { async function mounted() {

View File

@ -80,6 +80,7 @@ function curateRelease(release, type = 'scene', context = {}) {
curatedRelease.series = release.series?.filter(Boolean).map(({ serie }) => curateRelease(serie, 'serie', context)) || []; curatedRelease.series = release.series?.filter(Boolean).map(({ serie }) => curateRelease(serie, 'serie', context)) || [];
curatedRelease.chapters = release.chapters?.filter(Boolean).map((chapter) => curateRelease(chapter, 'chapter', context)) || []; curatedRelease.chapters = release.chapters?.filter(Boolean).map((chapter) => curateRelease(chapter, 'chapter', context)) || [];
curatedRelease.photos = release.photos?.filter(Boolean).map((photo) => photo.media || photo) || []; curatedRelease.photos = release.photos?.filter(Boolean).map((photo) => photo.media || photo) || [];
curatedRelease.caps = release.caps?.filter(Boolean).map((cap) => cap.media || cap) || [];
curatedRelease.scenesPhotos = release.scenesPhotos?.filter(Boolean).map((photo) => photo.media || photo) || []; curatedRelease.scenesPhotos = release.scenesPhotos?.filter(Boolean).map((photo) => photo.media || photo) || [];
curatedRelease.covers = release.covers?.filter(Boolean).map(({ media }) => media) || []; curatedRelease.covers = release.covers?.filter(Boolean).map(({ media }) => media) || [];

View File

@ -89,9 +89,20 @@ function initEntitiesActions(store, router) {
offset: $offset offset: $offset
orderBy: $orderBy orderBy: $orderBy
filter: { filter: {
not: { tags: { overlaps: $exclude } }
effectiveDate: { lessThan: $before, greaterThan: $after } effectiveDate: { lessThan: $before, greaterThan: $after }
showcased: { equalTo: true } showcased: { equalTo: true }
and: [
{
or: [
{
not: { tags: { overlaps: $exclude } }
}
{
tags: { isNull: true }
}
]
}
{
or: [ or: [
{ {
channelSlug: { equalTo: $entitySlug } channelSlug: { equalTo: $entitySlug }
@ -107,6 +118,8 @@ function initEntitiesActions(store, router) {
} }
] ]
} }
]
}
) { ) {
releases: nodes { releases: nodes {
release { release {

View File

@ -354,6 +354,31 @@ const releasePhotosFragment = `
} }
`; `;
const releaseCapsFragment = `
caps: releasesCaps(orderBy: MEDIA_BY_MEDIA_ID__INDEX_ASC) {
media {
id
index
path
thumbnail
width
height
thumbnailWidth
thumbnailHeight
lazy
isS3
comment
sfw: sfwMedia {
id
thumbnail
lazy
path
comment
}
}
}
`;
const releaseTrailerFragment = ` const releaseTrailerFragment = `
trailer: releasesTrailer { trailer: releasesTrailer {
media { media {
@ -398,6 +423,7 @@ const releaseFields = `
${releaseTagsFragment} ${releaseTagsFragment}
${releasePosterFragment} ${releasePosterFragment}
${releasePhotosFragment} ${releasePhotosFragment}
${releaseCapsFragment}
${siteFragment} ${siteFragment}
studio { studio {
id id
@ -470,7 +496,14 @@ const releasesFragment = `
offset: $offset offset: $offset
orderBy: $orderBy orderBy: $orderBy
filter: { filter: {
or: [
{
not: { tags: { overlaps: $exclude } } not: { tags: { overlaps: $exclude } }
}
{
tags: { isNull: true }
}
]
effectiveDate: { lessThan: $before, greaterThan: $after } effectiveDate: { lessThan: $before, greaterThan: $after }
showcased: { equalTo: true } showcased: { equalTo: true }
} }
@ -535,6 +568,7 @@ const releaseFragment = `
${releaseTagsFragment} ${releaseTagsFragment}
${releasePosterFragment} ${releasePosterFragment}
${releasePhotosFragment} ${releasePhotosFragment}
${releaseCapsFragment}
${releaseCoversFragment} ${releaseCoversFragment}
${releaseTrailerFragment} ${releaseTrailerFragment}
${releaseTeaserFragment} ${releaseTeaserFragment}

View File

@ -161,7 +161,14 @@ function initTagsActions(store, _router) {
offset: $offset offset: $offset
orderBy: $orderBy orderBy: $orderBy
filter: { filter: {
or: [
{
not: { tags: { overlaps: $exclude } } not: { tags: { overlaps: $exclude } }
}
{
tags: { isNull: true }
}
]
tags: { anyEqualTo: $tagSlug } tags: { anyEqualTo: $tagSlug }
effectiveDate: { lessThan: $before, greaterThan: $after } effectiveDate: { lessThan: $before, greaterThan: $after }
showcased: { equalTo: true } showcased: { equalTo: true }

View File

@ -0,0 +1,23 @@
const config = require('config');
exports.up = async (knex) => {
await knex.schema.createTable('releases_caps', (table) => {
table.integer('release_id')
.notNullable()
.references('id')
.inTable('releases');
table.text('media_id')
.notNullable()
.references('id')
.inTable('media');
});
await knex.raw('GRANT ALL ON releases_caps TO :visitor;', {
visitor: knex.raw(config.database.query.user),
});
};
exports.down = async (knex) => {
await knex.schema.dropTable('releases_caps');
};

View File

@ -0,0 +1,27 @@
exports.up = async (knex) => {
await knex.schema.alterTable('actors_profiles', (table) => {
table.string('hair_type');
table.decimal('shoe_size');
table.string('blood_type');
});
await knex.schema.alterTable('actors', (table) => {
table.string('hair_type');
table.decimal('shoe_size');
table.string('blood_type');
});
};
exports.down = async (knex) => {
await knex.schema.alterTable('actors_profiles', (table) => {
table.dropColumn('hair_type');
table.dropColumn('shoe_size');
table.dropColumn('blood_type');
});
await knex.schema.alterTable('actors', (table) => {
table.dropColumn('hair_type');
table.dropColumn('shoe_size');
table.dropColumn('blood_type');
});
};

14
package-lock.json generated
View File

@ -80,7 +80,7 @@
"tunnel": "0.0.6", "tunnel": "0.0.6",
"ua-parser-js": "^1.0.32", "ua-parser-js": "^1.0.32",
"undici": "^4.13.0", "undici": "^4.13.0",
"unprint": "^0.10.1", "unprint": "^0.10.3",
"url-pattern": "^1.0.3", "url-pattern": "^1.0.3",
"v-tooltip": "^2.0.3", "v-tooltip": "^2.0.3",
"video.js": "^7.11.4", "video.js": "^7.11.4",
@ -17538,9 +17538,9 @@
} }
}, },
"node_modules/unprint": { "node_modules/unprint": {
"version": "0.10.1", "version": "0.10.3",
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.10.1.tgz", "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.10.3.tgz",
"integrity": "sha512-2KtzIQKlOzXyDDyrCQQQXWuljC6kHjAhYZT1NRiDT2Lr1GgnwR+R9iVqbq6iz1Z1Oflt7ngpYW1MGHy3xDnduw==", "integrity": "sha512-ui8BbBo4JmKR++w50rSUFyg8X6l9EAbLRpATxdjxyS7yYevjcGMEt3HT0nrBG2JXDMkLwWZ+WoOaz3qC5stSxQ==",
"dependencies": { "dependencies": {
"axios": "^0.27.2", "axios": "^0.27.2",
"bottleneck": "^2.19.5", "bottleneck": "^2.19.5",
@ -32378,9 +32378,9 @@
"integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
}, },
"unprint": { "unprint": {
"version": "0.10.1", "version": "0.10.3",
"resolved": "https://registry.npmjs.org/unprint/-/unprint-0.10.1.tgz", "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.10.3.tgz",
"integrity": "sha512-2KtzIQKlOzXyDDyrCQQQXWuljC6kHjAhYZT1NRiDT2Lr1GgnwR+R9iVqbq6iz1Z1Oflt7ngpYW1MGHy3xDnduw==", "integrity": "sha512-ui8BbBo4JmKR++w50rSUFyg8X6l9EAbLRpATxdjxyS7yYevjcGMEt3HT0nrBG2JXDMkLwWZ+WoOaz3qC5stSxQ==",
"requires": { "requires": {
"axios": "^0.27.2", "axios": "^0.27.2",
"bottleneck": "^2.19.5", "bottleneck": "^2.19.5",

View File

@ -139,7 +139,7 @@
"tunnel": "0.0.6", "tunnel": "0.0.6",
"ua-parser-js": "^1.0.32", "ua-parser-js": "^1.0.32",
"undici": "^4.13.0", "undici": "^4.13.0",
"unprint": "^0.10.1", "unprint": "^0.10.3",
"url-pattern": "^1.0.3", "url-pattern": "^1.0.3",
"v-tooltip": "^2.0.3", "v-tooltip": "^2.0.3",
"video.js": "^7.11.4", "video.js": "^7.11.4",

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 12 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 12 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 12 KiB

View File

@ -1158,6 +1158,18 @@ const tags = [
name: 'exotic', name: 'exotic',
slug: 'exotic', slug: 'exotic',
}, },
{
name: 'japanese',
slug: 'japanese',
},
{
name: 'jav',
slug: 'jav',
},
{
name: 'fetish',
slug: 'fetish',
},
]; ];
const aliases = [ const aliases = [
@ -2287,11 +2299,23 @@ const aliases = [
}, },
{ {
name: 'pronebone', name: 'pronebone',
slug: 'prone-bone', for: 'prone-bone',
}, },
{ {
name: 'prone', name: 'prone',
slug: 'prone-bone', for: 'prone-bone',
},
{
name: 'japanese adult videos',
for: 'jav',
},
{
name: 'japanese adult video',
for: 'jav',
},
{
name: 'sm',
for: 'bdsm',
}, },
]; ];

View File

@ -11089,6 +11089,13 @@ const sites = [
siteId: 20, siteId: 20,
}, },
}, },
// TOKYO HOT
{
name: 'Tokyo Hot',
slug: 'tokyohot',
url: 'https://my.tokyo-hot.com',
tags: ['jav'],
},
// TOP WEB MODELS // TOP WEB MODELS
{ {
name: '2 Girls 1 Camera', name: '2 Girls 1 Camera',

View File

@ -106,6 +106,21 @@ const ethnicities = {
white: 'white', white: 'white',
}; };
const bloodTypes = {
A: 'A',
'A+': 'A+',
'A-': 'A-',
B: 'B',
'B+': 'B+',
'B-': 'B-',
AB: 'AB',
'AB+': 'AB+',
'AB-': 'AB-',
O: 'O',
'O+': 'O+',
'O-': 'O-',
};
function getBoolean(value) { function getBoolean(value) {
if (typeof value === 'boolean') { if (typeof value === 'boolean') {
return value; return value;
@ -195,6 +210,7 @@ function toBaseActors(actorsOrNames, release) {
name, name,
slug, slug,
entryId: (entity && (entryId || actorOrName.entryId)) || null, entryId: (entity && (entryId || actorOrName.entryId)) || null,
suppliedEntryId: entryId,
entity, entity,
hasProfile: !!actorOrName.name, // actor contains profile information hasProfile: !!actorOrName.name, // actor contains profile information
}; };
@ -257,12 +273,15 @@ function curateActor(actor, withDetails = false, isProfile = false) {
circumcised: actor.circumcised, circumcised: actor.circumcised,
height: actor.height, height: actor.height,
weight: actor.weight, weight: actor.weight,
shoeSize: actor.shoe_size,
eyes: actor.eyes, eyes: actor.eyes,
hairColor: actor.hair_color, hairColor: actor.hair_color,
hairType: actor.hair_type,
hasTattoos: actor.has_tattoos, hasTattoos: actor.has_tattoos,
hasPiercings: actor.has_piercings, hasPiercings: actor.has_piercings,
tattoos: actor.tattoos, tattoos: actor.tattoos,
piercings: actor.piercings, piercings: actor.piercings,
bloodType: actor.blood_type,
...(isProfile && { description: actor.description }), ...(isProfile && { description: actor.description }),
placeOfBirth: actor.birth_country && { placeOfBirth: actor.birth_country && {
country: { country: {
@ -347,12 +366,15 @@ function curateProfileEntry(profile) {
natural_boobs: profile.naturalBoobs, natural_boobs: profile.naturalBoobs,
height: profile.height, height: profile.height,
weight: profile.weight, weight: profile.weight,
shoe_size: profile.shoeSize,
hair_color: profile.hairColor, hair_color: profile.hairColor,
hair_type: profile.hairType,
eyes: profile.eyes, eyes: profile.eyes,
has_tattoos: profile.hasTattoos, has_tattoos: profile.hasTattoos,
has_piercings: profile.hasPiercings, has_piercings: profile.hasPiercings,
piercings: profile.piercings, piercings: profile.piercings,
tattoos: profile.tattoos, tattoos: profile.tattoos,
blood_type: profile.bloodType,
avatar_media_id: profile.avatarMediaId || null, avatar_media_id: profile.avatarMediaId || null,
}; };
@ -386,6 +408,7 @@ async function curateProfile(profile, actor) {
curatedProfile.nationality = profile.nationality?.trim() || null; // used to derive country when country not available curatedProfile.nationality = profile.nationality?.trim() || null; // used to derive country when country not available
curatedProfile.ethnicity = ethnicities[profile.ethnicity?.trim().toLowerCase()] || null; curatedProfile.ethnicity = ethnicities[profile.ethnicity?.trim().toLowerCase()] || null;
curatedProfile.hairType = profile.hairType?.trim() || null;
curatedProfile.hairColor = hairColors[(profile.hairColor || profile.hair)?.toLowerCase().replace('hair', '').trim()] || null; curatedProfile.hairColor = hairColors[(profile.hairColor || profile.hair)?.toLowerCase().replace('hair', '').trim()] || null;
curatedProfile.eyes = eyeColors[profile.eyes?.trim().toLowerCase()] || null; curatedProfile.eyes = eyeColors[profile.eyes?.trim().toLowerCase()] || null;
@ -411,6 +434,7 @@ async function curateProfile(profile, actor) {
curatedProfile.height = Number(profile.height) || profile.height?.match?.(/\d+/)?.[0] || null; curatedProfile.height = Number(profile.height) || profile.height?.match?.(/\d+/)?.[0] || null;
curatedProfile.weight = Number(profile.weight) || profile.weight?.match?.(/\d+/)?.[0] || null; curatedProfile.weight = Number(profile.weight) || profile.weight?.match?.(/\d+/)?.[0] || null;
curatedProfile.shoeSize = Number(profile.shoeSize) || profile.shoeSize?.match?.(/\d+/)?.[0] || null;
// separate measurement values // separate measurement values
curatedProfile.cup = profile.cup || (typeof profile.bust === 'string' && profile.bust?.match?.(/[a-zA-Z]+/)?.[0]) || null; curatedProfile.cup = profile.cup || (typeof profile.bust === 'string' && profile.bust?.match?.(/[a-zA-Z]+/)?.[0]) || null;
@ -435,6 +459,7 @@ async function curateProfile(profile, actor) {
curatedProfile.naturalBoobs = getBoolean(profile.naturalBoobs); curatedProfile.naturalBoobs = getBoolean(profile.naturalBoobs);
curatedProfile.hasTattoos = getBoolean(profile.hasTattoos); curatedProfile.hasTattoos = getBoolean(profile.hasTattoos);
curatedProfile.hasPiercings = getBoolean(profile.hasPiercings); curatedProfile.hasPiercings = getBoolean(profile.hasPiercings);
curatedProfile.bloodType = bloodTypes[profile.bloodType?.trim().toUpperCase()] || null;
if (argv.resolvePlace) { if (argv.resolvePlace) {
const [placeOfBirth, placeOfResidence] = await Promise.all([ const [placeOfBirth, placeOfResidence] = await Promise.all([
@ -564,6 +589,7 @@ async function interpolateProfiles(actorIdsOrNames) {
'bust', 'bust',
'waist', 'waist',
'hip', 'hip',
'shoe_size',
'penis_length', 'penis_length',
'penis_girth', 'penis_girth',
'circumcised', 'circumcised',
@ -571,6 +597,7 @@ async function interpolateProfiles(actorIdsOrNames) {
'eyes', 'eyes',
'has_tattoos', 'has_tattoos',
'has_piercings', 'has_piercings',
'blood_type',
].reduce((acc, property) => ({ ].reduce((acc, property) => ({
...acc, ...acc,
[property]: getMostFrequent(valuesByProperty[property]), [property]: getMostFrequent(valuesByProperty[property]),

View File

@ -16,7 +16,8 @@ const logger = require('./logger')(__filename);
const knex = require('./knex'); const knex = require('./knex');
const fetchUpdates = require('./updates'); const fetchUpdates = require('./updates');
const { fetchScenes, fetchMovies } = require('./deep'); const { fetchScenes, fetchMovies } = require('./deep');
const { storeScenes, storeMovies, updateSceneSearch, updateMovieSearch, associateMovieScenes } = require('./store-releases'); const { storeScenes, storeMovies, associateMovieScenes } = require('./store-releases');
const { updateSceneSearch, updateMovieSearch } = require('./update-search');
const { scrapeActors, deleteActors, flushActors, flushProfiles, interpolateProfiles } = require('./actors'); const { scrapeActors, deleteActors, flushActors, flushProfiles, interpolateProfiles } = require('./actors');
const { flushEntities } = require('./entities'); const { flushEntities } = require('./entities');
const { deleteScenes, deleteMovies, flushScenes, flushMovies, flushBatches } = require('./releases'); const { deleteScenes, deleteMovies, flushScenes, flushMovies, flushBatches } = require('./releases');

View File

@ -226,6 +226,11 @@ const { argv } = yargs
type: 'boolean', type: 'boolean',
default: true, default: true,
}) })
.option('caps', {
describe: 'Include release screen caps',
type: 'boolean',
default: true,
})
.option('trailers', { .option('trailers', {
describe: 'Include release trailers', describe: 'Include release trailers',
type: 'boolean', type: 'boolean',

View File

@ -567,7 +567,7 @@ async function storeFile(media, options) {
return storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath, options); return storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath, options);
} }
if (['posters', 'photos', 'covers'].includes(media.role)) { if (['posters', 'photos', 'caps', 'covers'].includes(media.role)) {
throw new Error(`Media for '${media.role}' must be an image, but '${media.meta.mimetype}' was detected`); throw new Error(`Media for '${media.role}' must be an image, but '${media.meta.mimetype}' was detected`);
} }
@ -873,6 +873,7 @@ async function associateReleaseMedia(releases, type = 'release') {
...(argv.images && argv.poster ? toBaseMedias([release.poster], 'posters') : []), ...(argv.images && argv.poster ? toBaseMedias([release.poster], 'posters') : []),
...(argv.images && argv.covers ? toBaseMedias(release.covers, 'covers') : []), ...(argv.images && argv.covers ? toBaseMedias(release.covers, 'covers') : []),
...(argv.images && argv.photos ? toBaseMedias(release.photos, 'photos') : []), ...(argv.images && argv.photos ? toBaseMedias(release.photos, 'photos') : []),
...(argv.images && argv.caps ? toBaseMedias(release.caps, 'caps') : []),
...(argv.videos && argv.trailer ? toBaseMedias([release.trailer], 'trailers') : []), ...(argv.videos && argv.trailer ? toBaseMedias([release.trailer], 'trailers') : []),
...(argv.videos && argv.teaser ? toBaseMedias([release.teaser], 'teasers') : []), ...(argv.videos && argv.teaser ? toBaseMedias([release.teaser], 'teasers') : []),
], ],
@ -888,7 +889,7 @@ async function associateReleaseMedia(releases, type = 'release') {
return acc; return acc;
}, {}); }, {});
await Promise.reduce(['posters', 'covers', 'photos', 'teasers', 'trailers'], async (chain, role) => { await Promise.reduce(['posters', 'covers', 'photos', 'caps', 'teasers', 'trailers'], async (chain, role) => {
// stage by role so posters are prioritized over photos and videos // stage by role so posters are prioritized over photos and videos
await chain; await chain;
@ -1006,6 +1007,7 @@ async function flushOrphanedMedia() {
knex('tags_photos').select('media_id'), knex('tags_photos').select('media_id'),
knex('releases_posters').select('media_id'), knex('releases_posters').select('media_id'),
knex('releases_photos').select('media_id'), knex('releases_photos').select('media_id'),
knex('releases_caps').select('media_id'),
knex('releases_covers').select('media_id'), knex('releases_covers').select('media_id'),
knex('releases_trailers').select('media_id'), knex('releases_trailers').select('media_id'),
knex('releases_teasers').select('media_id'), knex('releases_teasers').select('media_id'),

View File

@ -5,6 +5,7 @@ const inquirer = require('inquirer');
const logger = require('./logger')(__filename); const logger = require('./logger')(__filename);
const knex = require('./knex'); const knex = require('./knex');
const argv = require('./argv'); const argv = require('./argv');
const { updateSceneSearch } = require('./update-search');
const { flushOrphanedMedia } = require('./media'); const { flushOrphanedMedia } = require('./media');
const { graphql } = require('./web/graphql'); const { graphql } = require('./web/graphql');
@ -303,6 +304,8 @@ async function deleteScenes(sceneIds) {
.whereRaw('id = ANY(:sceneIds)', { sceneIds }) .whereRaw('id = ANY(:sceneIds)', { sceneIds })
.delete(); .delete();
await updateSceneSearch(sceneIds);
logger.info(`Removed ${deleteCount}/${sceneIds.length} scenes`); logger.info(`Removed ${deleteCount}/${sceneIds.length} scenes`);
return deleteCount; return deleteCount;

View File

@ -61,6 +61,7 @@ const spizoo = require('./spizoo');
const teamskeet = require('./teamskeet'); const teamskeet = require('./teamskeet');
const teencoreclub = require('./teencoreclub'); const teencoreclub = require('./teencoreclub');
const teenmegaworld = require('./teenmegaworld'); const teenmegaworld = require('./teenmegaworld');
const tokyohot = require('./tokyohot');
const topwebmodels = require('./topwebmodels'); const topwebmodels = require('./topwebmodels');
const traxxx = require('./traxxx'); const traxxx = require('./traxxx');
const vivid = require('./vivid'); const vivid = require('./vivid');
@ -151,6 +152,7 @@ const scrapers = {
teencoreclub, teencoreclub,
teenmegaworld, teenmegaworld,
teamskeet, teamskeet,
tokyohot,
topwebmodels, topwebmodels,
transbella: porndoe, transbella: porndoe,
traxxx, traxxx,
@ -288,6 +290,7 @@ const scrapers = {
teencoreclub, teencoreclub,
teenmegaworld, teenmegaworld,
thatsitcomshow: nubiles, thatsitcomshow: nubiles,
tokyohot,
topwebmodels, topwebmodels,
transangels: mindgeek, transangels: mindgeek,
transbella: porndoe, transbella: porndoe,

171
src/scrapers/tokyohot.js Normal file
View File

@ -0,0 +1,171 @@
'use strict';
const unprint = require('unprint');
const slugify = require('../utils/slugify');
function scrapeAll(scenes, channel) {
return scenes.map(({ query }) => {
const release = {};
const pathname = query.url();
release.url = unprint.prefixUrl(pathname, channel.url);
release.entryId = pathname.match(/product\/(\w+)/)?.[1];
release.shootId = query.attribute('img', 'title');
release.title = query.content('.title')?.replace(/^tokyo hot\s*/i, '');
release.description = query.content('.text');
const poster = query.img();
release.poster = [
poster.replace('220x124', '820x462'),
poster,
];
return release;
});
}
function scrapeScene({ query }, url, channel) {
const release = {};
release.entryId = new URL(url).pathname.match(/product\/(\w+)/)?.[1];
release.shootId = query.content('//dt[contains(text(), "Product ID")]/following-sibling::dd[1]');
release.title = query.content('.contents h2');
release.description = query.content('.contents .sentence');
release.date = query.date('//dt[contains(text(), "Release Date")]/following-sibling::dd[1]', 'YYYY/MM/DD');
release.duration = query.duration('//dt[contains(text(), "Duration")]/following-sibling::dd[1]');
release.actors = query.all('.info a[href*="/cast"]').map((el) => ({
name: unprint.query.content(el),
url: unprint.query.url(el, null, { origin: channel.url }),
}));
release.tags = query.contents('.info a[href*="type=play"]');
const poster = query.poster('.movie video');
release.poster = [
poster,
poster.replace('820x462', '220x124'),
];
release.trailer = query.video('.movie source');
release.photos = query.imgs('.scap a', { attribute: 'href' }).map((img) => [
img,
img.replace('640x480_wlimited', '150x150_default'),
]);
release.caps = query.imgs('.vcap a', { attribute: 'href' }).map((img) => [
img,
img.replace('640x480_wlimited', '120x120_default'),
]);
return release;
}
// measurements are specified as a range in centimeters 85 ~ 89cm
function getMeasurement(string, inches = false) {
if (!string) {
return null;
}
const value = Array.from(string.matchAll(/(\d+(?:\.\d+)?)\s*cm/g)).at(-1)?.[1];
if (!value) {
return null;
}
if (inches) {
return Math.round(Number(value) * 0.393701);
}
return Number(value);
}
function scrapeProfile({ query }) {
const profile = {};
const keys = query.contents('.info dt');
const values = query.contents('.info dd');
const bio = Object.fromEntries(keys.map((key, index) => [slugify(key, '_'), values[index]]));
profile.birthPlace = bio.home_town;
profile.height = getMeasurement(bio.height);
profile.cup = bio.cup_size?.replace('cup', '').trim();
profile.bust = getMeasurement(bio.bust_size, true);
profile.waist = getMeasurement(bio.waist_size, true);
profile.hip = getMeasurement(bio.hip_size || bio.hip, true);
profile.hairStyle = bio.hair_style;
profile.shoeSize = getMeasurement(bio.shoes_size);
profile.bloodType = bio.blood_type.replace('type', '').trim();
profile.avatar = query.img('#profile img');
return profile;
}
async function fetchLatest(channel, page) {
const url = `${channel.url}/product/?vendor=Tokyo-Hot&page=${page}&order=published_at`;
const res = await unprint.get(url, {
selectAll: '#main .list .detail',
agent: {
rejectUnauthorized: false,
},
});
if (res.ok) {
return scrapeAll(res.context, channel);
}
return res.status;
}
async function fetchScene(url, channel) {
const res = await unprint.get(url, {
agent: {
rejectUnauthorized: false,
},
});
if (res.ok) {
return scrapeScene(res.context, url, channel);
}
return res.status;
}
async function fetchProfile(actor, context) {
if (!actor.url) {
// search is cumbersome
return null;
}
const res = await unprint.get(actor.url, {
agent: {
rejectUnauthorized: false,
},
});
if (res.ok) {
return scrapeProfile(res.context, context);
}
return res.status;
}
module.exports = {
fetchLatest,
fetchScene,
fetchProfile,
};

View File

@ -16,6 +16,7 @@ const { associateActors, associateDirectors, scrapeActors, toBaseActors } = requ
const { associateReleaseTags } = require('./tags'); const { associateReleaseTags } = require('./tags');
const { curateEntity } = require('./entities'); const { curateEntity } = require('./entities');
const { associateReleaseMedia } = require('./media'); const { associateReleaseMedia } = require('./media');
const { updateSceneSearch, updateMovieSearch } = require('./update-search');
const { notify } = require('./alerts'); const { notify } = require('./alerts');
async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') { async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
@ -229,50 +230,6 @@ async function filterDuplicateReleases(releases) {
}; };
} }
async function updateSceneSearch(releaseIds) {
logger.info(`Updating search documents for ${releaseIds ? releaseIds.length : 'all' } releases`);
const documents = await knex.raw(`
SELECT
releases.id AS release_id,
TO_TSVECTOR(
'english',
COALESCE(releases.title, '') || ' ' ||
releases.entry_id || ' ' ||
entities.name || ' ' ||
entities.slug || ' ' ||
COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
COALESCE(parents.name, '') || ' ' ||
COALESCE(parents.slug, '') || ' ' ||
COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
COALESCE(releases.shoot_id, '') || ' ' ||
COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(directors.name, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(tags.name, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(tags_aliases.name, ''), ' ')
) as document
FROM releases
LEFT JOIN entities ON releases.entity_id = entities.id
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
LEFT JOIN actors ON local_actors.actor_id = actors.id
LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
${releaseIds ? 'WHERE releases.id = ANY(?)' : ''}
GROUP BY releases.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
`, releaseIds && [releaseIds]);
if (documents.rows?.length > 0) {
await bulkInsert('releases_search', documents.rows, ['release_id']);
}
await knex.raw('REFRESH MATERIALIZED VIEW releases_summaries;');
}
async function storeChapters(releases) { async function storeChapters(releases) {
const chapters = releases const chapters = releases
.map((release) => release.chapters?.map((chapter, index) => ({ .map((release) => release.chapters?.map((chapter, index) => ({
@ -380,44 +337,6 @@ async function associateSerieScenes(series, serieScenes) {
await bulkInsert('series_scenes', associations, false); await bulkInsert('series_scenes', associations, false);
} }
async function updateMovieSearch(movieIds, target = 'movie') {
logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } ${target}s`);
const documents = await knex.raw(`
SELECT
${target}s.id AS ${target}_id,
TO_TSVECTOR(
'english',
COALESCE(${target}s.title, '') || ' ' ||
entities.name || ' ' ||
entities.slug || ' ' ||
COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
COALESCE(parents.name, '') || ' ' ||
COALESCE(parents.slug, '') || ' ' ||
COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
COALESCE(TO_CHAR(${target}s.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(tags.name, ''), ' ')
) as document
FROM ${target}s
LEFT JOIN entities ON ${target}s.entity_id = entities.id
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
LEFT JOIN ${target}s_scenes ON ${target}s_scenes.${target}_id = ${target}s.id
LEFT JOIN releases ON releases.id = ${target}s_scenes.scene_id
LEFT JOIN releases_actors ON releases_actors.release_id = ${target}s_scenes.scene_id
LEFT JOIN releases_tags ON releases_tags.release_id = releases.id
LEFT JOIN actors ON actors.id = releases_actors.actor_id
LEFT JOIN tags ON tags.id = releases_tags.tag_id
${movieIds ? `WHERE ${target}s.id = ANY(?)` : ''}
GROUP BY ${target}s.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
`, movieIds && [movieIds]);
if (documents.rows?.length > 0) {
await bulkInsert(`${target}s_search`, documents.rows, [`${target}_id`]);
}
}
async function storeMovies(movies, useBatchId) { async function storeMovies(movies, useBatchId) {
if (!movies || movies.length === 0) { if (!movies || movies.length === 0) {
return []; return [];

92
src/update-search.js Normal file
View File

@ -0,0 +1,92 @@
'use strict';
const knex = require('./knex');
const logger = require('./logger')(__filename);
const bulkInsert = require('./utils/bulk-insert');
async function updateSceneSearch(releaseIds) {
logger.info(`Updating search documents for ${releaseIds ? releaseIds.length : 'all' } releases`);
const documents = await knex.raw(`
SELECT
releases.id AS release_id,
TO_TSVECTOR(
'english',
COALESCE(releases.title, '') || ' ' ||
releases.entry_id || ' ' ||
entities.name || ' ' ||
entities.slug || ' ' ||
COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
COALESCE(parents.name, '') || ' ' ||
COALESCE(parents.slug, '') || ' ' ||
COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
COALESCE(releases.shoot_id, '') || ' ' ||
COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(directors.name, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(tags.name, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(tags_aliases.name, ''), ' ')
) as document
FROM releases
LEFT JOIN entities ON releases.entity_id = entities.id
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
LEFT JOIN actors ON local_actors.actor_id = actors.id
LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6
LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
${releaseIds ? 'WHERE releases.id = ANY(?)' : ''}
GROUP BY releases.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
`, releaseIds && [releaseIds]);
if (documents.rows?.length > 0) {
await bulkInsert('releases_search', documents.rows, ['release_id']);
}
await knex.raw('REFRESH MATERIALIZED VIEW releases_summaries;');
}
async function updateMovieSearch(movieIds, target = 'movie') {
logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } ${target}s`);
const documents = await knex.raw(`
SELECT
${target}s.id AS ${target}_id,
TO_TSVECTOR(
'english',
COALESCE(${target}s.title, '') || ' ' ||
entities.name || ' ' ||
entities.slug || ' ' ||
COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
COALESCE(parents.name, '') || ' ' ||
COALESCE(parents.slug, '') || ' ' ||
COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
COALESCE(TO_CHAR(${target}s.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
STRING_AGG(COALESCE(tags.name, ''), ' ')
) as document
FROM ${target}s
LEFT JOIN entities ON ${target}s.entity_id = entities.id
LEFT JOIN entities AS parents ON parents.id = entities.parent_id
LEFT JOIN ${target}s_scenes ON ${target}s_scenes.${target}_id = ${target}s.id
LEFT JOIN releases ON releases.id = ${target}s_scenes.scene_id
LEFT JOIN releases_actors ON releases_actors.release_id = ${target}s_scenes.scene_id
LEFT JOIN releases_tags ON releases_tags.release_id = releases.id
LEFT JOIN actors ON actors.id = releases_actors.actor_id
LEFT JOIN tags ON tags.id = releases_tags.tag_id
${movieIds ? `WHERE ${target}s.id = ANY(?)` : ''}
GROUP BY ${target}s.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
`, movieIds && [movieIds]);
if (documents.rows?.length > 0) {
await bulkInsert(`${target}s_search`, documents.rows, [`${target}_id`]);
}
}
module.exports = {
updateSceneSearch,
updateMovieSearch,
};