({ ...acc, [clip.poster.id]: clip.poster }), {});
const uniqueClipPosters = Array.from(new Set(clips.map((clip) => clip.poster.id) || [])).map((posterId) => clipPostersById[posterId]);
- const photosWithClipPosters = (this.release.photos || []).concat(this.release.scenesPhotos || []).concat(uniqueClipPosters);
+ const photosWithClipPosters = (this.release.photos || []).concat(this.release.caps || []).concat(this.release.scenesPhotos || []).concat(uniqueClipPosters);
if (this.release.trailer || (this.release.teaser && this.release.teaser.mime !== 'image/gif')) {
// poster will be on trailer video
diff --git a/assets/components/releases/release.vue b/assets/components/releases/release.vue
index 4a6e6173..9a1c322e 100755
--- a/assets/components/releases/release.vue
+++ b/assets/components/releases/release.vue
@@ -21,14 +21,14 @@
0 || this.release.scenesPhotos?.length > 0) && this.$route.hash === '#album';
+ return this.release.photos?.length > 0 || this.release.caps?.length > 0 || this.release.scenesPhotos?.length > 0;
}
async function mounted() {
diff --git a/assets/js/curate.js b/assets/js/curate.js
index ece3535f..ac135cb3 100755
--- a/assets/js/curate.js
+++ b/assets/js/curate.js
@@ -80,6 +80,7 @@ function curateRelease(release, type = 'scene', context = {}) {
curatedRelease.series = release.series?.filter(Boolean).map(({ serie }) => curateRelease(serie, 'serie', context)) || [];
curatedRelease.chapters = release.chapters?.filter(Boolean).map((chapter) => curateRelease(chapter, 'chapter', context)) || [];
curatedRelease.photos = release.photos?.filter(Boolean).map((photo) => photo.media || photo) || [];
+ curatedRelease.caps = release.caps?.filter(Boolean).map((cap) => cap.media || cap) || [];
curatedRelease.scenesPhotos = release.scenesPhotos?.filter(Boolean).map((photo) => photo.media || photo) || [];
curatedRelease.covers = release.covers?.filter(Boolean).map(({ media }) => media) || [];
diff --git a/assets/js/entities/actions.js b/assets/js/entities/actions.js
index 07b392ac..0a59e502 100755
--- a/assets/js/entities/actions.js
+++ b/assets/js/entities/actions.js
@@ -89,21 +89,34 @@ function initEntitiesActions(store, router) {
offset: $offset
orderBy: $orderBy
filter: {
- not: { tags: { overlaps: $exclude } }
effectiveDate: { lessThan: $before, greaterThan: $after }
showcased: { equalTo: true }
- or: [
+ and: [
{
- channelSlug: { equalTo: $entitySlug }
- channelType: { equalTo: $entityType }
+ or: [
+ {
+ not: { tags: { overlaps: $exclude } }
+ }
+ {
+ tags: { isNull: true }
+ }
+ ]
}
{
- networkSlug: { equalTo: $entitySlug }
- networkType: { equalTo: $entityType }
- }
- {
- parentNetworkSlug: { equalTo: $entitySlug }
- parentNetworkType: { equalTo: $entityType }
+ or: [
+ {
+ channelSlug: { equalTo: $entitySlug }
+ channelType: { equalTo: $entityType }
+ }
+ {
+ networkSlug: { equalTo: $entitySlug }
+ networkType: { equalTo: $entityType }
+ }
+ {
+ parentNetworkSlug: { equalTo: $entitySlug }
+ parentNetworkType: { equalTo: $entityType }
+ }
+ ]
}
]
}
diff --git a/assets/js/fragments.js b/assets/js/fragments.js
index fcd8a3c6..06483210 100755
--- a/assets/js/fragments.js
+++ b/assets/js/fragments.js
@@ -354,6 +354,31 @@ const releasePhotosFragment = `
}
`;
+const releaseCapsFragment = `
+ caps: releasesCaps(orderBy: MEDIA_BY_MEDIA_ID__INDEX_ASC) {
+ media {
+ id
+ index
+ path
+ thumbnail
+ width
+ height
+ thumbnailWidth
+ thumbnailHeight
+ lazy
+ isS3
+ comment
+ sfw: sfwMedia {
+ id
+ thumbnail
+ lazy
+ path
+ comment
+ }
+ }
+ }
+`;
+
const releaseTrailerFragment = `
trailer: releasesTrailer {
media {
@@ -398,6 +423,7 @@ const releaseFields = `
${releaseTagsFragment}
${releasePosterFragment}
${releasePhotosFragment}
+ ${releaseCapsFragment}
${siteFragment}
studio {
id
@@ -470,7 +496,14 @@ const releasesFragment = `
offset: $offset
orderBy: $orderBy
filter: {
- not: { tags: { overlaps: $exclude } }
+ or: [
+ {
+ not: { tags: { overlaps: $exclude } }
+ }
+ {
+ tags: { isNull: true }
+ }
+ ]
effectiveDate: { lessThan: $before, greaterThan: $after }
showcased: { equalTo: true }
}
@@ -535,6 +568,7 @@ const releaseFragment = `
${releaseTagsFragment}
${releasePosterFragment}
${releasePhotosFragment}
+ ${releaseCapsFragment}
${releaseCoversFragment}
${releaseTrailerFragment}
${releaseTeaserFragment}
diff --git a/assets/js/tags/actions.js b/assets/js/tags/actions.js
index 21c2ce15..1868111d 100755
--- a/assets/js/tags/actions.js
+++ b/assets/js/tags/actions.js
@@ -161,7 +161,14 @@ function initTagsActions(store, _router) {
offset: $offset
orderBy: $orderBy
filter: {
- not: { tags: { overlaps: $exclude } }
+ or: [
+ {
+ not: { tags: { overlaps: $exclude } }
+ }
+ {
+ tags: { isNull: true }
+ }
+ ]
tags: { anyEqualTo: $tagSlug }
effectiveDate: { lessThan: $before, greaterThan: $after }
showcased: { equalTo: true }
diff --git a/migrations/20230725001453_caps.js b/migrations/20230725001453_caps.js
new file mode 100644
index 00000000..9bac273c
--- /dev/null
+++ b/migrations/20230725001453_caps.js
@@ -0,0 +1,23 @@
+const config = require('config');
+
+exports.up = async (knex) => {
+ await knex.schema.createTable('releases_caps', (table) => {
+ table.integer('release_id')
+ .notNullable()
+ .references('id')
+ .inTable('releases');
+
+ table.text('media_id')
+ .notNullable()
+ .references('id')
+ .inTable('media');
+ });
+
+ await knex.raw('GRANT ALL ON releases_caps TO :visitor;', {
+ visitor: knex.raw(config.database.query.user),
+ });
+};
+
+exports.down = async (knex) => {
+ await knex.schema.dropTable('releases_caps');
+};
diff --git a/migrations/20230725020639_blood_type.js b/migrations/20230725020639_blood_type.js
new file mode 100644
index 00000000..af1a36d9
--- /dev/null
+++ b/migrations/20230725020639_blood_type.js
@@ -0,0 +1,27 @@
+exports.up = async (knex) => {
+ await knex.schema.alterTable('actors_profiles', (table) => {
+ table.string('hair_type');
+ table.decimal('shoe_size');
+ table.string('blood_type');
+ });
+
+ await knex.schema.alterTable('actors', (table) => {
+ table.string('hair_type');
+ table.decimal('shoe_size');
+ table.string('blood_type');
+ });
+};
+
+exports.down = async (knex) => {
+ await knex.schema.alterTable('actors_profiles', (table) => {
+ table.dropColumn('hair_type');
+ table.dropColumn('shoe_size');
+ table.dropColumn('blood_type');
+ });
+
+ await knex.schema.alterTable('actors', (table) => {
+ table.dropColumn('hair_type');
+ table.dropColumn('shoe_size');
+ table.dropColumn('blood_type');
+ });
+};
diff --git a/package-lock.json b/package-lock.json
index 1ebeffbf..f264f1a7 100755
--- a/package-lock.json
+++ b/package-lock.json
@@ -80,7 +80,7 @@
"tunnel": "0.0.6",
"ua-parser-js": "^1.0.32",
"undici": "^4.13.0",
- "unprint": "^0.10.1",
+ "unprint": "^0.10.3",
"url-pattern": "^1.0.3",
"v-tooltip": "^2.0.3",
"video.js": "^7.11.4",
@@ -17538,9 +17538,9 @@
}
},
"node_modules/unprint": {
- "version": "0.10.1",
- "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.10.1.tgz",
- "integrity": "sha512-2KtzIQKlOzXyDDyrCQQQXWuljC6kHjAhYZT1NRiDT2Lr1GgnwR+R9iVqbq6iz1Z1Oflt7ngpYW1MGHy3xDnduw==",
+ "version": "0.10.3",
+ "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.10.3.tgz",
+ "integrity": "sha512-ui8BbBo4JmKR++w50rSUFyg8X6l9EAbLRpATxdjxyS7yYevjcGMEt3HT0nrBG2JXDMkLwWZ+WoOaz3qC5stSxQ==",
"dependencies": {
"axios": "^0.27.2",
"bottleneck": "^2.19.5",
@@ -32378,9 +32378,9 @@
"integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
},
"unprint": {
- "version": "0.10.1",
- "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.10.1.tgz",
- "integrity": "sha512-2KtzIQKlOzXyDDyrCQQQXWuljC6kHjAhYZT1NRiDT2Lr1GgnwR+R9iVqbq6iz1Z1Oflt7ngpYW1MGHy3xDnduw==",
+ "version": "0.10.3",
+ "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.10.3.tgz",
+ "integrity": "sha512-ui8BbBo4JmKR++w50rSUFyg8X6l9EAbLRpATxdjxyS7yYevjcGMEt3HT0nrBG2JXDMkLwWZ+WoOaz3qC5stSxQ==",
"requires": {
"axios": "^0.27.2",
"bottleneck": "^2.19.5",
diff --git a/package.json b/package.json
index f49d7588..a21c3327 100755
--- a/package.json
+++ b/package.json
@@ -139,7 +139,7 @@
"tunnel": "0.0.6",
"ua-parser-js": "^1.0.32",
"undici": "^4.13.0",
- "unprint": "^0.10.1",
+ "unprint": "^0.10.3",
"url-pattern": "^1.0.3",
"v-tooltip": "^2.0.3",
"video.js": "^7.11.4",
diff --git a/public/img/logos/tokyohot/favicon.png b/public/img/logos/tokyohot/favicon.png
new file mode 100644
index 00000000..69c41bef
Binary files /dev/null and b/public/img/logos/tokyohot/favicon.png differ
diff --git a/public/img/logos/tokyohot/favicon_dark.png b/public/img/logos/tokyohot/favicon_dark.png
new file mode 100644
index 00000000..dbd4b0f3
Binary files /dev/null and b/public/img/logos/tokyohot/favicon_dark.png differ
diff --git a/public/img/logos/tokyohot/favicon_light.png b/public/img/logos/tokyohot/favicon_light.png
new file mode 100644
index 00000000..592daa53
Binary files /dev/null and b/public/img/logos/tokyohot/favicon_light.png differ
diff --git a/public/img/logos/tokyohot/lazy/network.png b/public/img/logos/tokyohot/lazy/network.png
new file mode 100644
index 00000000..aa7d73fa
Binary files /dev/null and b/public/img/logos/tokyohot/lazy/network.png differ
diff --git a/public/img/logos/tokyohot/lazy/tokyohot.png b/public/img/logos/tokyohot/lazy/tokyohot.png
new file mode 100644
index 00000000..2ea36427
Binary files /dev/null and b/public/img/logos/tokyohot/lazy/tokyohot.png differ
diff --git a/public/img/logos/tokyohot/misc/tokyo-hot.png b/public/img/logos/tokyohot/misc/tokyo-hot.png
new file mode 100644
index 00000000..b0e8a070
Binary files /dev/null and b/public/img/logos/tokyohot/misc/tokyo-hot.png differ
diff --git a/public/img/logos/tokyohot/misc/tokyo-hot_white.png b/public/img/logos/tokyohot/misc/tokyo-hot_white.png
new file mode 100644
index 00000000..650f7bc3
Binary files /dev/null and b/public/img/logos/tokyohot/misc/tokyo-hot_white.png differ
diff --git a/public/img/logos/tokyohot/network.png b/public/img/logos/tokyohot/network.png
new file mode 100644
index 00000000..30c71208
Binary files /dev/null and b/public/img/logos/tokyohot/network.png differ
diff --git a/public/img/logos/tokyohot/thumbs/network.png b/public/img/logos/tokyohot/thumbs/network.png
new file mode 100644
index 00000000..9d3d69f7
Binary files /dev/null and b/public/img/logos/tokyohot/thumbs/network.png differ
diff --git a/public/img/logos/tokyohot/thumbs/tokyohot.png b/public/img/logos/tokyohot/thumbs/tokyohot.png
new file mode 100644
index 00000000..f9a695bd
Binary files /dev/null and b/public/img/logos/tokyohot/thumbs/tokyohot.png differ
diff --git a/public/img/logos/tokyohot/tokyohot.png b/public/img/logos/tokyohot/tokyohot.png
new file mode 100644
index 00000000..c7a913c9
Binary files /dev/null and b/public/img/logos/tokyohot/tokyohot.png differ
diff --git a/seeds/00_tags.js b/seeds/00_tags.js
index df09cc4c..244f1791 100755
--- a/seeds/00_tags.js
+++ b/seeds/00_tags.js
@@ -1158,6 +1158,18 @@ const tags = [
name: 'exotic',
slug: 'exotic',
},
+ {
+ name: 'japanese',
+ slug: 'japanese',
+ },
+ {
+ name: 'jav',
+ slug: 'jav',
+ },
+ {
+ name: 'fetish',
+ slug: 'fetish',
+ },
];
const aliases = [
@@ -2287,11 +2299,23 @@ const aliases = [
},
{
name: 'pronebone',
- slug: 'prone-bone',
+ for: 'prone-bone',
},
{
name: 'prone',
- slug: 'prone-bone',
+ for: 'prone-bone',
+ },
+ {
+ name: 'japanese adult videos',
+ for: 'jav',
+ },
+ {
+ name: 'japanese adult video',
+ for: 'jav',
+ },
+ {
+ name: 'sm',
+ for: 'bdsm',
},
];
diff --git a/seeds/02_sites.js b/seeds/02_sites.js
index 9ba58728..59cc0c76 100755
--- a/seeds/02_sites.js
+++ b/seeds/02_sites.js
@@ -11089,6 +11089,13 @@ const sites = [
siteId: 20,
},
},
+ // TOKYO HOT
+ {
+ name: 'Tokyo Hot',
+ slug: 'tokyohot',
+ url: 'https://my.tokyo-hot.com',
+ tags: ['jav'],
+ },
// TOP WEB MODELS
{
name: '2 Girls 1 Camera',
diff --git a/src/actors.js b/src/actors.js
index 4ea8f77e..192c604a 100755
--- a/src/actors.js
+++ b/src/actors.js
@@ -106,6 +106,21 @@ const ethnicities = {
white: 'white',
};
+const bloodTypes = {
+ A: 'A',
+ 'A+': 'A+',
+ 'A-': 'A-',
+ B: 'B',
+ 'B+': 'B+',
+ 'B-': 'B-',
+ AB: 'AB',
+ 'AB+': 'AB+',
+ 'AB-': 'AB-',
+ O: 'O',
+ 'O+': 'O+',
+ 'O-': 'O-',
+};
+
function getBoolean(value) {
if (typeof value === 'boolean') {
return value;
@@ -195,6 +210,7 @@ function toBaseActors(actorsOrNames, release) {
name,
slug,
entryId: (entity && (entryId || actorOrName.entryId)) || null,
+ suppliedEntryId: entryId,
entity,
hasProfile: !!actorOrName.name, // actor contains profile information
};
@@ -257,12 +273,15 @@ function curateActor(actor, withDetails = false, isProfile = false) {
circumcised: actor.circumcised,
height: actor.height,
weight: actor.weight,
+ shoeSize: actor.shoe_size,
eyes: actor.eyes,
hairColor: actor.hair_color,
+ hairType: actor.hair_type,
hasTattoos: actor.has_tattoos,
hasPiercings: actor.has_piercings,
tattoos: actor.tattoos,
piercings: actor.piercings,
+ bloodType: actor.blood_type,
...(isProfile && { description: actor.description }),
placeOfBirth: actor.birth_country && {
country: {
@@ -347,12 +366,15 @@ function curateProfileEntry(profile) {
natural_boobs: profile.naturalBoobs,
height: profile.height,
weight: profile.weight,
+ shoe_size: profile.shoeSize,
hair_color: profile.hairColor,
+ hair_type: profile.hairType,
eyes: profile.eyes,
has_tattoos: profile.hasTattoos,
has_piercings: profile.hasPiercings,
piercings: profile.piercings,
tattoos: profile.tattoos,
+ blood_type: profile.bloodType,
avatar_media_id: profile.avatarMediaId || null,
};
@@ -386,6 +408,7 @@ async function curateProfile(profile, actor) {
curatedProfile.nationality = profile.nationality?.trim() || null; // used to derive country when country not available
curatedProfile.ethnicity = ethnicities[profile.ethnicity?.trim().toLowerCase()] || null;
+ curatedProfile.hairType = profile.hairType?.trim() || null;
curatedProfile.hairColor = hairColors[(profile.hairColor || profile.hair)?.toLowerCase().replace('hair', '').trim()] || null;
curatedProfile.eyes = eyeColors[profile.eyes?.trim().toLowerCase()] || null;
@@ -411,6 +434,7 @@ async function curateProfile(profile, actor) {
curatedProfile.height = Number(profile.height) || profile.height?.match?.(/\d+/)?.[0] || null;
curatedProfile.weight = Number(profile.weight) || profile.weight?.match?.(/\d+/)?.[0] || null;
+ curatedProfile.shoeSize = Number(profile.shoeSize) || profile.shoeSize?.match?.(/\d+/)?.[0] || null;
// separate measurement values
curatedProfile.cup = profile.cup || (typeof profile.bust === 'string' && profile.bust?.match?.(/[a-zA-Z]+/)?.[0]) || null;
@@ -435,6 +459,7 @@ async function curateProfile(profile, actor) {
curatedProfile.naturalBoobs = getBoolean(profile.naturalBoobs);
curatedProfile.hasTattoos = getBoolean(profile.hasTattoos);
curatedProfile.hasPiercings = getBoolean(profile.hasPiercings);
+ curatedProfile.bloodType = bloodTypes[profile.bloodType?.trim().toUpperCase()] || null;
if (argv.resolvePlace) {
const [placeOfBirth, placeOfResidence] = await Promise.all([
@@ -564,6 +589,7 @@ async function interpolateProfiles(actorIdsOrNames) {
'bust',
'waist',
'hip',
+ 'shoe_size',
'penis_length',
'penis_girth',
'circumcised',
@@ -571,6 +597,7 @@ async function interpolateProfiles(actorIdsOrNames) {
'eyes',
'has_tattoos',
'has_piercings',
+ 'blood_type',
].reduce((acc, property) => ({
...acc,
[property]: getMostFrequent(valuesByProperty[property]),
diff --git a/src/app.js b/src/app.js
index a5d3bd4c..c07f3cf4 100755
--- a/src/app.js
+++ b/src/app.js
@@ -16,7 +16,8 @@ const logger = require('./logger')(__filename);
const knex = require('./knex');
const fetchUpdates = require('./updates');
const { fetchScenes, fetchMovies } = require('./deep');
-const { storeScenes, storeMovies, updateSceneSearch, updateMovieSearch, associateMovieScenes } = require('./store-releases');
+const { storeScenes, storeMovies, associateMovieScenes } = require('./store-releases');
+const { updateSceneSearch, updateMovieSearch } = require('./update-search');
const { scrapeActors, deleteActors, flushActors, flushProfiles, interpolateProfiles } = require('./actors');
const { flushEntities } = require('./entities');
const { deleteScenes, deleteMovies, flushScenes, flushMovies, flushBatches } = require('./releases');
diff --git a/src/argv.js b/src/argv.js
index ff2bc4b6..d4df5df2 100755
--- a/src/argv.js
+++ b/src/argv.js
@@ -226,6 +226,11 @@ const { argv } = yargs
type: 'boolean',
default: true,
})
+ .option('caps', {
+ describe: 'Include release screen caps',
+ type: 'boolean',
+ default: true,
+ })
.option('trailers', {
describe: 'Include release trailers',
type: 'boolean',
diff --git a/src/media.js b/src/media.js
index b3164ad5..cddc2635 100755
--- a/src/media.js
+++ b/src/media.js
@@ -567,7 +567,7 @@ async function storeFile(media, options) {
return storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath, options);
}
- if (['posters', 'photos', 'covers'].includes(media.role)) {
+ if (['posters', 'photos', 'caps', 'covers'].includes(media.role)) {
throw new Error(`Media for '${media.role}' must be an image, but '${media.meta.mimetype}' was detected`);
}
@@ -873,6 +873,7 @@ async function associateReleaseMedia(releases, type = 'release') {
...(argv.images && argv.poster ? toBaseMedias([release.poster], 'posters') : []),
...(argv.images && argv.covers ? toBaseMedias(release.covers, 'covers') : []),
...(argv.images && argv.photos ? toBaseMedias(release.photos, 'photos') : []),
+ ...(argv.images && argv.caps ? toBaseMedias(release.caps, 'caps') : []),
...(argv.videos && argv.trailer ? toBaseMedias([release.trailer], 'trailers') : []),
...(argv.videos && argv.teaser ? toBaseMedias([release.teaser], 'teasers') : []),
],
@@ -888,7 +889,7 @@ async function associateReleaseMedia(releases, type = 'release') {
return acc;
}, {});
- await Promise.reduce(['posters', 'covers', 'photos', 'teasers', 'trailers'], async (chain, role) => {
+ await Promise.reduce(['posters', 'covers', 'photos', 'caps', 'teasers', 'trailers'], async (chain, role) => {
// stage by role so posters are prioritized over photos and videos
await chain;
@@ -1006,6 +1007,7 @@ async function flushOrphanedMedia() {
knex('tags_photos').select('media_id'),
knex('releases_posters').select('media_id'),
knex('releases_photos').select('media_id'),
+ knex('releases_caps').select('media_id'),
knex('releases_covers').select('media_id'),
knex('releases_trailers').select('media_id'),
knex('releases_teasers').select('media_id'),
diff --git a/src/releases.js b/src/releases.js
index 1ebf78bf..4e636b43 100755
--- a/src/releases.js
+++ b/src/releases.js
@@ -5,6 +5,7 @@ const inquirer = require('inquirer');
const logger = require('./logger')(__filename);
const knex = require('./knex');
const argv = require('./argv');
+const { updateSceneSearch } = require('./update-search');
const { flushOrphanedMedia } = require('./media');
const { graphql } = require('./web/graphql');
@@ -303,6 +304,8 @@ async function deleteScenes(sceneIds) {
.whereRaw('id = ANY(:sceneIds)', { sceneIds })
.delete();
+ await updateSceneSearch(sceneIds);
+
logger.info(`Removed ${deleteCount}/${sceneIds.length} scenes`);
return deleteCount;
diff --git a/src/scrapers/scrapers.js b/src/scrapers/scrapers.js
index d9f6b6c7..b075c73c 100755
--- a/src/scrapers/scrapers.js
+++ b/src/scrapers/scrapers.js
@@ -61,6 +61,7 @@ const spizoo = require('./spizoo');
const teamskeet = require('./teamskeet');
const teencoreclub = require('./teencoreclub');
const teenmegaworld = require('./teenmegaworld');
+const tokyohot = require('./tokyohot');
const topwebmodels = require('./topwebmodels');
const traxxx = require('./traxxx');
const vivid = require('./vivid');
@@ -151,6 +152,7 @@ const scrapers = {
teencoreclub,
teenmegaworld,
teamskeet,
+ tokyohot,
topwebmodels,
transbella: porndoe,
traxxx,
@@ -288,6 +290,7 @@ const scrapers = {
teencoreclub,
teenmegaworld,
thatsitcomshow: nubiles,
+ tokyohot,
topwebmodels,
transangels: mindgeek,
transbella: porndoe,
diff --git a/src/scrapers/tokyohot.js b/src/scrapers/tokyohot.js
new file mode 100644
index 00000000..b8f1e05d
--- /dev/null
+++ b/src/scrapers/tokyohot.js
@@ -0,0 +1,171 @@
+'use strict';
+
+const unprint = require('unprint');
+
+const slugify = require('../utils/slugify');
+
+function scrapeAll(scenes, channel) {
+ return scenes.map(({ query }) => {
+ const release = {};
+
+ const pathname = query.url();
+
+ release.url = unprint.prefixUrl(pathname, channel.url);
+ release.entryId = pathname.match(/product\/(\w+)/)?.[1];
+ release.shootId = query.attribute('img', 'title');
+
+ release.title = query.content('.title')?.replace(/^tokyo hot\s*/i, '');
+ release.description = query.content('.text');
+
+ const poster = query.img();
+
+ release.poster = [
+ poster.replace('220x124', '820x462'),
+ poster,
+ ];
+
+ return release;
+ });
+}
+
+function scrapeScene({ query }, url, channel) {
+ const release = {};
+
+ release.entryId = new URL(url).pathname.match(/product\/(\w+)/)?.[1];
+ release.shootId = query.content('//dt[contains(text(), "Product ID")]/following-sibling::dd[1]');
+
+ release.title = query.content('.contents h2');
+ release.description = query.content('.contents .sentence');
+ release.date = query.date('//dt[contains(text(), "Release Date")]/following-sibling::dd[1]', 'YYYY/MM/DD');
+ release.duration = query.duration('//dt[contains(text(), "Duration")]/following-sibling::dd[1]');
+
+ release.actors = query.all('.info a[href*="/cast"]').map((el) => ({
+ name: unprint.query.content(el),
+ url: unprint.query.url(el, null, { origin: channel.url }),
+ }));
+
+ release.tags = query.contents('.info a[href*="type=play"]');
+
+ const poster = query.poster('.movie video');
+
+ release.poster = [
+ poster,
+ poster.replace('820x462', '220x124'),
+ ];
+
+ release.trailer = query.video('.movie source');
+
+ release.photos = query.imgs('.scap a', { attribute: 'href' }).map((img) => [
+ img,
+ img.replace('640x480_wlimited', '150x150_default'),
+ ]);
+
+ release.caps = query.imgs('.vcap a', { attribute: 'href' }).map((img) => [
+ img,
+ img.replace('640x480_wlimited', '120x120_default'),
+ ]);
+
+ return release;
+}
+
+// measurements are specified as a range in centimeters 85 ~ 89cm
+function getMeasurement(string, inches = false) {
+ if (!string) {
+ return null;
+ }
+
+ const value = Array.from(string.matchAll(/(\d+(?:\.\d+)?)\s*cm/g)).at(-1)?.[1];
+
+ if (!value) {
+ return null;
+ }
+
+ if (inches) {
+ return Math.round(Number(value) * 0.393701);
+ }
+
+ return Number(value);
+}
+
+function scrapeProfile({ query }) {
+ const profile = {};
+
+ const keys = query.contents('.info dt');
+ const values = query.contents('.info dd');
+
+ const bio = Object.fromEntries(keys.map((key, index) => [slugify(key, '_'), values[index]]));
+
+ profile.birthPlace = bio.home_town;
+
+ profile.height = getMeasurement(bio.height);
+
+ profile.cup = bio.cup_size?.replace('cup', '').trim();
+ profile.bust = getMeasurement(bio.bust_size, true);
+ profile.waist = getMeasurement(bio.waist_size, true);
+ profile.hip = getMeasurement(bio.hip_size || bio.hip, true);
+
+ profile.hairStyle = bio.hair_style;
+ profile.shoeSize = getMeasurement(bio.shoes_size);
+
+ profile.bloodType = bio.blood_type.replace('type', '').trim();
+
+ profile.avatar = query.img('#profile img');
+
+ return profile;
+}
+
+async function fetchLatest(channel, page) {
+ const url = `${channel.url}/product/?vendor=Tokyo-Hot&page=${page}&order=published_at`;
+
+ const res = await unprint.get(url, {
+ selectAll: '#main .list .detail',
+ agent: {
+ rejectUnauthorized: false,
+ },
+ });
+
+ if (res.ok) {
+ return scrapeAll(res.context, channel);
+ }
+
+ return res.status;
+}
+
+async function fetchScene(url, channel) {
+ const res = await unprint.get(url, {
+ agent: {
+ rejectUnauthorized: false,
+ },
+ });
+
+ if (res.ok) {
+ return scrapeScene(res.context, url, channel);
+ }
+
+ return res.status;
+}
+
+async function fetchProfile(actor, context) {
+ if (!actor.url) {
+ // search is cumbersome
+ return null;
+ }
+
+ const res = await unprint.get(actor.url, {
+ agent: {
+ rejectUnauthorized: false,
+ },
+ });
+
+ if (res.ok) {
+ return scrapeProfile(res.context, context);
+ }
+
+ return res.status;
+}
+
+module.exports = {
+ fetchLatest,
+ fetchScene,
+ fetchProfile,
+};
diff --git a/src/store-releases.js b/src/store-releases.js
index 44dabc54..8697eb85 100755
--- a/src/store-releases.js
+++ b/src/store-releases.js
@@ -16,6 +16,7 @@ const { associateActors, associateDirectors, scrapeActors, toBaseActors } = requ
const { associateReleaseTags } = require('./tags');
const { curateEntity } = require('./entities');
const { associateReleaseMedia } = require('./media');
+const { updateSceneSearch, updateMovieSearch } = require('./update-search');
const { notify } = require('./alerts');
async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
@@ -229,50 +230,6 @@ async function filterDuplicateReleases(releases) {
};
}
-async function updateSceneSearch(releaseIds) {
- logger.info(`Updating search documents for ${releaseIds ? releaseIds.length : 'all' } releases`);
-
- const documents = await knex.raw(`
- SELECT
- releases.id AS release_id,
- TO_TSVECTOR(
- 'english',
- COALESCE(releases.title, '') || ' ' ||
- releases.entry_id || ' ' ||
- entities.name || ' ' ||
- entities.slug || ' ' ||
- COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
- COALESCE(parents.name, '') || ' ' ||
- COALESCE(parents.slug, '') || ' ' ||
- COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
- COALESCE(releases.shoot_id, '') || ' ' ||
- COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
- STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
- STRING_AGG(COALESCE(directors.name, ''), ' ') || ' ' ||
- STRING_AGG(COALESCE(tags.name, ''), ' ') || ' ' ||
- STRING_AGG(COALESCE(tags_aliases.name, ''), ' ')
- ) as document
- FROM releases
- LEFT JOIN entities ON releases.entity_id = entities.id
- LEFT JOIN entities AS parents ON parents.id = entities.parent_id
- LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
- LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
- LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
- LEFT JOIN actors ON local_actors.actor_id = actors.id
- LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
- LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6
- LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
- ${releaseIds ? 'WHERE releases.id = ANY(?)' : ''}
- GROUP BY releases.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
- `, releaseIds && [releaseIds]);
-
- if (documents.rows?.length > 0) {
- await bulkInsert('releases_search', documents.rows, ['release_id']);
- }
-
- await knex.raw('REFRESH MATERIALIZED VIEW releases_summaries;');
-}
-
async function storeChapters(releases) {
const chapters = releases
.map((release) => release.chapters?.map((chapter, index) => ({
@@ -380,44 +337,6 @@ async function associateSerieScenes(series, serieScenes) {
await bulkInsert('series_scenes', associations, false);
}
-async function updateMovieSearch(movieIds, target = 'movie') {
- logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } ${target}s`);
-
- const documents = await knex.raw(`
- SELECT
- ${target}s.id AS ${target}_id,
- TO_TSVECTOR(
- 'english',
- COALESCE(${target}s.title, '') || ' ' ||
- entities.name || ' ' ||
- entities.slug || ' ' ||
- COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
- COALESCE(parents.name, '') || ' ' ||
- COALESCE(parents.slug, '') || ' ' ||
- COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
- COALESCE(TO_CHAR(${target}s.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
- STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' ||
- STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
- STRING_AGG(COALESCE(tags.name, ''), ' ')
- ) as document
- FROM ${target}s
- LEFT JOIN entities ON ${target}s.entity_id = entities.id
- LEFT JOIN entities AS parents ON parents.id = entities.parent_id
- LEFT JOIN ${target}s_scenes ON ${target}s_scenes.${target}_id = ${target}s.id
- LEFT JOIN releases ON releases.id = ${target}s_scenes.scene_id
- LEFT JOIN releases_actors ON releases_actors.release_id = ${target}s_scenes.scene_id
- LEFT JOIN releases_tags ON releases_tags.release_id = releases.id
- LEFT JOIN actors ON actors.id = releases_actors.actor_id
- LEFT JOIN tags ON tags.id = releases_tags.tag_id
- ${movieIds ? `WHERE ${target}s.id = ANY(?)` : ''}
- GROUP BY ${target}s.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
- `, movieIds && [movieIds]);
-
- if (documents.rows?.length > 0) {
- await bulkInsert(`${target}s_search`, documents.rows, [`${target}_id`]);
- }
-}
-
async function storeMovies(movies, useBatchId) {
if (!movies || movies.length === 0) {
return [];
diff --git a/src/update-search.js b/src/update-search.js
new file mode 100644
index 00000000..bf03dd5d
--- /dev/null
+++ b/src/update-search.js
@@ -0,0 +1,92 @@
+'use strict';
+
+const knex = require('./knex');
+const logger = require('./logger')(__filename);
+const bulkInsert = require('./utils/bulk-insert');
+
+async function updateSceneSearch(releaseIds) {
+ logger.info(`Updating search documents for ${releaseIds ? releaseIds.length : 'all' } releases`);
+
+ const documents = await knex.raw(`
+ SELECT
+ releases.id AS release_id,
+ TO_TSVECTOR(
+ 'english',
+ COALESCE(releases.title, '') || ' ' ||
+ releases.entry_id || ' ' ||
+ entities.name || ' ' ||
+ entities.slug || ' ' ||
+ COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
+ COALESCE(parents.name, '') || ' ' ||
+ COALESCE(parents.slug, '') || ' ' ||
+ COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
+ COALESCE(releases.shoot_id, '') || ' ' ||
+ COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
+ STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
+ STRING_AGG(COALESCE(directors.name, ''), ' ') || ' ' ||
+ STRING_AGG(COALESCE(tags.name, ''), ' ') || ' ' ||
+ STRING_AGG(COALESCE(tags_aliases.name, ''), ' ')
+ ) as document
+ FROM releases
+ LEFT JOIN entities ON releases.entity_id = entities.id
+ LEFT JOIN entities AS parents ON parents.id = entities.parent_id
+ LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id
+ LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id
+ LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id
+ LEFT JOIN actors ON local_actors.actor_id = actors.id
+ LEFT JOIN actors AS directors ON local_directors.director_id = directors.id
+ LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6
+ LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true
+ ${releaseIds ? 'WHERE releases.id = ANY(?)' : ''}
+ GROUP BY releases.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
+ `, releaseIds && [releaseIds]);
+
+ if (documents.rows?.length > 0) {
+ await bulkInsert('releases_search', documents.rows, ['release_id']);
+ }
+
+ await knex.raw('REFRESH MATERIALIZED VIEW releases_summaries;');
+}
+
+async function updateMovieSearch(movieIds, target = 'movie') {
+ logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } ${target}s`);
+
+ const documents = await knex.raw(`
+ SELECT
+ ${target}s.id AS ${target}_id,
+ TO_TSVECTOR(
+ 'english',
+ COALESCE(${target}s.title, '') || ' ' ||
+ entities.name || ' ' ||
+ entities.slug || ' ' ||
+ COALESCE(array_to_string(entities.alias, ' '), '') || ' ' ||
+ COALESCE(parents.name, '') || ' ' ||
+ COALESCE(parents.slug, '') || ' ' ||
+ COALESCE(array_to_string(parents.alias, ' '), '') || ' ' ||
+ COALESCE(TO_CHAR(${target}s.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' ||
+ STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' ||
+ STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' ||
+ STRING_AGG(COALESCE(tags.name, ''), ' ')
+ ) as document
+ FROM ${target}s
+ LEFT JOIN entities ON ${target}s.entity_id = entities.id
+ LEFT JOIN entities AS parents ON parents.id = entities.parent_id
+ LEFT JOIN ${target}s_scenes ON ${target}s_scenes.${target}_id = ${target}s.id
+ LEFT JOIN releases ON releases.id = ${target}s_scenes.scene_id
+ LEFT JOIN releases_actors ON releases_actors.release_id = ${target}s_scenes.scene_id
+ LEFT JOIN releases_tags ON releases_tags.release_id = releases.id
+ LEFT JOIN actors ON actors.id = releases_actors.actor_id
+ LEFT JOIN tags ON tags.id = releases_tags.tag_id
+ ${movieIds ? `WHERE ${target}s.id = ANY(?)` : ''}
+ GROUP BY ${target}s.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias;
+ `, movieIds && [movieIds]);
+
+ if (documents.rows?.length > 0) {
+ await bulkInsert(`${target}s_search`, documents.rows, [`${target}_id`]);
+ }
+}
+
+module.exports = {
+ updateSceneSearch,
+ updateMovieSearch,
+};