diff --git a/assets/components/releases/banner.vue b/assets/components/releases/banner.vue index bef85578c..afee58e53 100755 --- a/assets/components/releases/banner.vue +++ b/assets/components/releases/banner.vue @@ -2,7 +2,7 @@
({ ...acc, [clip.poster.id]: clip.poster }), {}); const uniqueClipPosters = Array.from(new Set(clips.map((clip) => clip.poster.id) || [])).map((posterId) => clipPostersById[posterId]); - const photosWithClipPosters = (this.release.photos || []).concat(this.release.scenesPhotos || []).concat(uniqueClipPosters); + const photosWithClipPosters = (this.release.photos || []).concat(this.release.caps || []).concat(this.release.scenesPhotos || []).concat(uniqueClipPosters); if (this.release.trailer || (this.release.teaser && this.release.teaser.mime !== 'image/gif')) { // poster will be on trailer video diff --git a/assets/components/releases/release.vue b/assets/components/releases/release.vue index 4a6e6173a..9a1c322e5 100755 --- a/assets/components/releases/release.vue +++ b/assets/components/releases/release.vue @@ -21,14 +21,14 @@
0 || this.release.scenesPhotos?.length > 0) && this.$route.hash === '#album'; + return this.release.photos?.length > 0 || this.release.caps?.length > 0 || this.release.scenesPhotos?.length > 0; } async function mounted() { diff --git a/assets/js/curate.js b/assets/js/curate.js index ece3535f7..ac135cb39 100755 --- a/assets/js/curate.js +++ b/assets/js/curate.js @@ -80,6 +80,7 @@ function curateRelease(release, type = 'scene', context = {}) { curatedRelease.series = release.series?.filter(Boolean).map(({ serie }) => curateRelease(serie, 'serie', context)) || []; curatedRelease.chapters = release.chapters?.filter(Boolean).map((chapter) => curateRelease(chapter, 'chapter', context)) || []; curatedRelease.photos = release.photos?.filter(Boolean).map((photo) => photo.media || photo) || []; + curatedRelease.caps = release.caps?.filter(Boolean).map((cap) => cap.media || cap) || []; curatedRelease.scenesPhotos = release.scenesPhotos?.filter(Boolean).map((photo) => photo.media || photo) || []; curatedRelease.covers = release.covers?.filter(Boolean).map(({ media }) => media) || []; diff --git a/assets/js/entities/actions.js b/assets/js/entities/actions.js index 07b392ac3..0a59e502a 100755 --- a/assets/js/entities/actions.js +++ b/assets/js/entities/actions.js @@ -89,21 +89,34 @@ function initEntitiesActions(store, router) { offset: $offset orderBy: $orderBy filter: { - not: { tags: { overlaps: $exclude } } effectiveDate: { lessThan: $before, greaterThan: $after } showcased: { equalTo: true } - or: [ + and: [ { - channelSlug: { equalTo: $entitySlug } - channelType: { equalTo: $entityType } + or: [ + { + not: { tags: { overlaps: $exclude } } + } + { + tags: { isNull: true } + } + ] } { - networkSlug: { equalTo: $entitySlug } - networkType: { equalTo: $entityType } - } - { - parentNetworkSlug: { equalTo: $entitySlug } - parentNetworkType: { equalTo: $entityType } + or: [ + { + channelSlug: { equalTo: $entitySlug } + channelType: { equalTo: $entityType } + } + { + networkSlug: { equalTo: $entitySlug } + networkType: { equalTo: $entityType } + } + { + parentNetworkSlug: { equalTo: $entitySlug } + parentNetworkType: { equalTo: $entityType } + } + ] } ] } diff --git a/assets/js/fragments.js b/assets/js/fragments.js index fcd8a3c6e..064832108 100755 --- a/assets/js/fragments.js +++ b/assets/js/fragments.js @@ -354,6 +354,31 @@ const releasePhotosFragment = ` } `; +const releaseCapsFragment = ` + caps: releasesCaps(orderBy: MEDIA_BY_MEDIA_ID__INDEX_ASC) { + media { + id + index + path + thumbnail + width + height + thumbnailWidth + thumbnailHeight + lazy + isS3 + comment + sfw: sfwMedia { + id + thumbnail + lazy + path + comment + } + } + } +`; + const releaseTrailerFragment = ` trailer: releasesTrailer { media { @@ -398,6 +423,7 @@ const releaseFields = ` ${releaseTagsFragment} ${releasePosterFragment} ${releasePhotosFragment} + ${releaseCapsFragment} ${siteFragment} studio { id @@ -470,7 +496,14 @@ const releasesFragment = ` offset: $offset orderBy: $orderBy filter: { - not: { tags: { overlaps: $exclude } } + or: [ + { + not: { tags: { overlaps: $exclude } } + } + { + tags: { isNull: true } + } + ] effectiveDate: { lessThan: $before, greaterThan: $after } showcased: { equalTo: true } } @@ -535,6 +568,7 @@ const releaseFragment = ` ${releaseTagsFragment} ${releasePosterFragment} ${releasePhotosFragment} + ${releaseCapsFragment} ${releaseCoversFragment} ${releaseTrailerFragment} ${releaseTeaserFragment} diff --git a/assets/js/tags/actions.js b/assets/js/tags/actions.js index 21c2ce15a..1868111d4 100755 --- a/assets/js/tags/actions.js +++ b/assets/js/tags/actions.js @@ -161,7 +161,14 @@ function initTagsActions(store, _router) { offset: $offset orderBy: $orderBy filter: { - not: { tags: { overlaps: $exclude } } + or: [ + { + not: { tags: { overlaps: $exclude } } + } + { + tags: { isNull: true } + } + ] tags: { anyEqualTo: $tagSlug } effectiveDate: { lessThan: $before, greaterThan: $after } showcased: { equalTo: true } diff --git a/migrations/20230725001453_caps.js b/migrations/20230725001453_caps.js new file mode 100644 index 000000000..9bac273cc --- /dev/null +++ b/migrations/20230725001453_caps.js @@ -0,0 +1,23 @@ +const config = require('config'); + +exports.up = async (knex) => { + await knex.schema.createTable('releases_caps', (table) => { + table.integer('release_id') + .notNullable() + .references('id') + .inTable('releases'); + + table.text('media_id') + .notNullable() + .references('id') + .inTable('media'); + }); + + await knex.raw('GRANT ALL ON releases_caps TO :visitor;', { + visitor: knex.raw(config.database.query.user), + }); +}; + +exports.down = async (knex) => { + await knex.schema.dropTable('releases_caps'); +}; diff --git a/migrations/20230725020639_blood_type.js b/migrations/20230725020639_blood_type.js new file mode 100644 index 000000000..af1a36d9a --- /dev/null +++ b/migrations/20230725020639_blood_type.js @@ -0,0 +1,27 @@ +exports.up = async (knex) => { + await knex.schema.alterTable('actors_profiles', (table) => { + table.string('hair_type'); + table.decimal('shoe_size'); + table.string('blood_type'); + }); + + await knex.schema.alterTable('actors', (table) => { + table.string('hair_type'); + table.decimal('shoe_size'); + table.string('blood_type'); + }); +}; + +exports.down = async (knex) => { + await knex.schema.alterTable('actors_profiles', (table) => { + table.dropColumn('hair_type'); + table.dropColumn('shoe_size'); + table.dropColumn('blood_type'); + }); + + await knex.schema.alterTable('actors', (table) => { + table.dropColumn('hair_type'); + table.dropColumn('shoe_size'); + table.dropColumn('blood_type'); + }); +}; diff --git a/package-lock.json b/package-lock.json index 1ebeffbf3..f264f1a78 100755 --- a/package-lock.json +++ b/package-lock.json @@ -80,7 +80,7 @@ "tunnel": "0.0.6", "ua-parser-js": "^1.0.32", "undici": "^4.13.0", - "unprint": "^0.10.1", + "unprint": "^0.10.3", "url-pattern": "^1.0.3", "v-tooltip": "^2.0.3", "video.js": "^7.11.4", @@ -17538,9 +17538,9 @@ } }, "node_modules/unprint": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.10.1.tgz", - "integrity": "sha512-2KtzIQKlOzXyDDyrCQQQXWuljC6kHjAhYZT1NRiDT2Lr1GgnwR+R9iVqbq6iz1Z1Oflt7ngpYW1MGHy3xDnduw==", + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.10.3.tgz", + "integrity": "sha512-ui8BbBo4JmKR++w50rSUFyg8X6l9EAbLRpATxdjxyS7yYevjcGMEt3HT0nrBG2JXDMkLwWZ+WoOaz3qC5stSxQ==", "dependencies": { "axios": "^0.27.2", "bottleneck": "^2.19.5", @@ -32378,9 +32378,9 @@ "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" }, "unprint": { - "version": "0.10.1", - "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.10.1.tgz", - "integrity": "sha512-2KtzIQKlOzXyDDyrCQQQXWuljC6kHjAhYZT1NRiDT2Lr1GgnwR+R9iVqbq6iz1Z1Oflt7ngpYW1MGHy3xDnduw==", + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/unprint/-/unprint-0.10.3.tgz", + "integrity": "sha512-ui8BbBo4JmKR++w50rSUFyg8X6l9EAbLRpATxdjxyS7yYevjcGMEt3HT0nrBG2JXDMkLwWZ+WoOaz3qC5stSxQ==", "requires": { "axios": "^0.27.2", "bottleneck": "^2.19.5", diff --git a/package.json b/package.json index f49d75889..a21c3327f 100755 --- a/package.json +++ b/package.json @@ -139,7 +139,7 @@ "tunnel": "0.0.6", "ua-parser-js": "^1.0.32", "undici": "^4.13.0", - "unprint": "^0.10.1", + "unprint": "^0.10.3", "url-pattern": "^1.0.3", "v-tooltip": "^2.0.3", "video.js": "^7.11.4", diff --git a/public/img/logos/tokyohot/favicon.png b/public/img/logos/tokyohot/favicon.png new file mode 100644 index 000000000..69c41bef0 Binary files /dev/null and b/public/img/logos/tokyohot/favicon.png differ diff --git a/public/img/logos/tokyohot/favicon_dark.png b/public/img/logos/tokyohot/favicon_dark.png new file mode 100644 index 000000000..dbd4b0f3b Binary files /dev/null and b/public/img/logos/tokyohot/favicon_dark.png differ diff --git a/public/img/logos/tokyohot/favicon_light.png b/public/img/logos/tokyohot/favicon_light.png new file mode 100644 index 000000000..592daa535 Binary files /dev/null and b/public/img/logos/tokyohot/favicon_light.png differ diff --git a/public/img/logos/tokyohot/lazy/network.png b/public/img/logos/tokyohot/lazy/network.png new file mode 100644 index 000000000..aa7d73fa2 Binary files /dev/null and b/public/img/logos/tokyohot/lazy/network.png differ diff --git a/public/img/logos/tokyohot/lazy/tokyohot.png b/public/img/logos/tokyohot/lazy/tokyohot.png new file mode 100644 index 000000000..2ea36427a Binary files /dev/null and b/public/img/logos/tokyohot/lazy/tokyohot.png differ diff --git a/public/img/logos/tokyohot/misc/tokyo-hot.png b/public/img/logos/tokyohot/misc/tokyo-hot.png new file mode 100644 index 000000000..b0e8a0700 Binary files /dev/null and b/public/img/logos/tokyohot/misc/tokyo-hot.png differ diff --git a/public/img/logos/tokyohot/misc/tokyo-hot_white.png b/public/img/logos/tokyohot/misc/tokyo-hot_white.png new file mode 100644 index 000000000..650f7bc36 Binary files /dev/null and b/public/img/logos/tokyohot/misc/tokyo-hot_white.png differ diff --git a/public/img/logos/tokyohot/network.png b/public/img/logos/tokyohot/network.png new file mode 100644 index 000000000..30c712081 Binary files /dev/null and b/public/img/logos/tokyohot/network.png differ diff --git a/public/img/logos/tokyohot/thumbs/network.png b/public/img/logos/tokyohot/thumbs/network.png new file mode 100644 index 000000000..9d3d69f7c Binary files /dev/null and b/public/img/logos/tokyohot/thumbs/network.png differ diff --git a/public/img/logos/tokyohot/thumbs/tokyohot.png b/public/img/logos/tokyohot/thumbs/tokyohot.png new file mode 100644 index 000000000..f9a695bdb Binary files /dev/null and b/public/img/logos/tokyohot/thumbs/tokyohot.png differ diff --git a/public/img/logos/tokyohot/tokyohot.png b/public/img/logos/tokyohot/tokyohot.png new file mode 100644 index 000000000..c7a913c93 Binary files /dev/null and b/public/img/logos/tokyohot/tokyohot.png differ diff --git a/seeds/00_tags.js b/seeds/00_tags.js index df09cc4c6..244f1791a 100755 --- a/seeds/00_tags.js +++ b/seeds/00_tags.js @@ -1158,6 +1158,18 @@ const tags = [ name: 'exotic', slug: 'exotic', }, + { + name: 'japanese', + slug: 'japanese', + }, + { + name: 'jav', + slug: 'jav', + }, + { + name: 'fetish', + slug: 'fetish', + }, ]; const aliases = [ @@ -2287,11 +2299,23 @@ const aliases = [ }, { name: 'pronebone', - slug: 'prone-bone', + for: 'prone-bone', }, { name: 'prone', - slug: 'prone-bone', + for: 'prone-bone', + }, + { + name: 'japanese adult videos', + for: 'jav', + }, + { + name: 'japanese adult video', + for: 'jav', + }, + { + name: 'sm', + for: 'bdsm', }, ]; diff --git a/seeds/02_sites.js b/seeds/02_sites.js index 9ba58728c..59cc0c760 100755 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -11089,6 +11089,13 @@ const sites = [ siteId: 20, }, }, + // TOKYO HOT + { + name: 'Tokyo Hot', + slug: 'tokyohot', + url: 'https://my.tokyo-hot.com', + tags: ['jav'], + }, // TOP WEB MODELS { name: '2 Girls 1 Camera', diff --git a/src/actors.js b/src/actors.js index 4ea8f77e1..192c604ab 100755 --- a/src/actors.js +++ b/src/actors.js @@ -106,6 +106,21 @@ const ethnicities = { white: 'white', }; +const bloodTypes = { + A: 'A', + 'A+': 'A+', + 'A-': 'A-', + B: 'B', + 'B+': 'B+', + 'B-': 'B-', + AB: 'AB', + 'AB+': 'AB+', + 'AB-': 'AB-', + O: 'O', + 'O+': 'O+', + 'O-': 'O-', +}; + function getBoolean(value) { if (typeof value === 'boolean') { return value; @@ -195,6 +210,7 @@ function toBaseActors(actorsOrNames, release) { name, slug, entryId: (entity && (entryId || actorOrName.entryId)) || null, + suppliedEntryId: entryId, entity, hasProfile: !!actorOrName.name, // actor contains profile information }; @@ -257,12 +273,15 @@ function curateActor(actor, withDetails = false, isProfile = false) { circumcised: actor.circumcised, height: actor.height, weight: actor.weight, + shoeSize: actor.shoe_size, eyes: actor.eyes, hairColor: actor.hair_color, + hairType: actor.hair_type, hasTattoos: actor.has_tattoos, hasPiercings: actor.has_piercings, tattoos: actor.tattoos, piercings: actor.piercings, + bloodType: actor.blood_type, ...(isProfile && { description: actor.description }), placeOfBirth: actor.birth_country && { country: { @@ -347,12 +366,15 @@ function curateProfileEntry(profile) { natural_boobs: profile.naturalBoobs, height: profile.height, weight: profile.weight, + shoe_size: profile.shoeSize, hair_color: profile.hairColor, + hair_type: profile.hairType, eyes: profile.eyes, has_tattoos: profile.hasTattoos, has_piercings: profile.hasPiercings, piercings: profile.piercings, tattoos: profile.tattoos, + blood_type: profile.bloodType, avatar_media_id: profile.avatarMediaId || null, }; @@ -386,6 +408,7 @@ async function curateProfile(profile, actor) { curatedProfile.nationality = profile.nationality?.trim() || null; // used to derive country when country not available curatedProfile.ethnicity = ethnicities[profile.ethnicity?.trim().toLowerCase()] || null; + curatedProfile.hairType = profile.hairType?.trim() || null; curatedProfile.hairColor = hairColors[(profile.hairColor || profile.hair)?.toLowerCase().replace('hair', '').trim()] || null; curatedProfile.eyes = eyeColors[profile.eyes?.trim().toLowerCase()] || null; @@ -411,6 +434,7 @@ async function curateProfile(profile, actor) { curatedProfile.height = Number(profile.height) || profile.height?.match?.(/\d+/)?.[0] || null; curatedProfile.weight = Number(profile.weight) || profile.weight?.match?.(/\d+/)?.[0] || null; + curatedProfile.shoeSize = Number(profile.shoeSize) || profile.shoeSize?.match?.(/\d+/)?.[0] || null; // separate measurement values curatedProfile.cup = profile.cup || (typeof profile.bust === 'string' && profile.bust?.match?.(/[a-zA-Z]+/)?.[0]) || null; @@ -435,6 +459,7 @@ async function curateProfile(profile, actor) { curatedProfile.naturalBoobs = getBoolean(profile.naturalBoobs); curatedProfile.hasTattoos = getBoolean(profile.hasTattoos); curatedProfile.hasPiercings = getBoolean(profile.hasPiercings); + curatedProfile.bloodType = bloodTypes[profile.bloodType?.trim().toUpperCase()] || null; if (argv.resolvePlace) { const [placeOfBirth, placeOfResidence] = await Promise.all([ @@ -564,6 +589,7 @@ async function interpolateProfiles(actorIdsOrNames) { 'bust', 'waist', 'hip', + 'shoe_size', 'penis_length', 'penis_girth', 'circumcised', @@ -571,6 +597,7 @@ async function interpolateProfiles(actorIdsOrNames) { 'eyes', 'has_tattoos', 'has_piercings', + 'blood_type', ].reduce((acc, property) => ({ ...acc, [property]: getMostFrequent(valuesByProperty[property]), diff --git a/src/app.js b/src/app.js index a5d3bd4c9..c07f3cf40 100755 --- a/src/app.js +++ b/src/app.js @@ -16,7 +16,8 @@ const logger = require('./logger')(__filename); const knex = require('./knex'); const fetchUpdates = require('./updates'); const { fetchScenes, fetchMovies } = require('./deep'); -const { storeScenes, storeMovies, updateSceneSearch, updateMovieSearch, associateMovieScenes } = require('./store-releases'); +const { storeScenes, storeMovies, associateMovieScenes } = require('./store-releases'); +const { updateSceneSearch, updateMovieSearch } = require('./update-search'); const { scrapeActors, deleteActors, flushActors, flushProfiles, interpolateProfiles } = require('./actors'); const { flushEntities } = require('./entities'); const { deleteScenes, deleteMovies, flushScenes, flushMovies, flushBatches } = require('./releases'); diff --git a/src/argv.js b/src/argv.js index ff2bc4b69..d4df5df2b 100755 --- a/src/argv.js +++ b/src/argv.js @@ -226,6 +226,11 @@ const { argv } = yargs type: 'boolean', default: true, }) + .option('caps', { + describe: 'Include release screen caps', + type: 'boolean', + default: true, + }) .option('trailers', { describe: 'Include release trailers', type: 'boolean', diff --git a/src/media.js b/src/media.js index b3164ad5e..cddc2635d 100755 --- a/src/media.js +++ b/src/media.js @@ -567,7 +567,7 @@ async function storeFile(media, options) { return storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath, options); } - if (['posters', 'photos', 'covers'].includes(media.role)) { + if (['posters', 'photos', 'caps', 'covers'].includes(media.role)) { throw new Error(`Media for '${media.role}' must be an image, but '${media.meta.mimetype}' was detected`); } @@ -873,6 +873,7 @@ async function associateReleaseMedia(releases, type = 'release') { ...(argv.images && argv.poster ? toBaseMedias([release.poster], 'posters') : []), ...(argv.images && argv.covers ? toBaseMedias(release.covers, 'covers') : []), ...(argv.images && argv.photos ? toBaseMedias(release.photos, 'photos') : []), + ...(argv.images && argv.caps ? toBaseMedias(release.caps, 'caps') : []), ...(argv.videos && argv.trailer ? toBaseMedias([release.trailer], 'trailers') : []), ...(argv.videos && argv.teaser ? toBaseMedias([release.teaser], 'teasers') : []), ], @@ -888,7 +889,7 @@ async function associateReleaseMedia(releases, type = 'release') { return acc; }, {}); - await Promise.reduce(['posters', 'covers', 'photos', 'teasers', 'trailers'], async (chain, role) => { + await Promise.reduce(['posters', 'covers', 'photos', 'caps', 'teasers', 'trailers'], async (chain, role) => { // stage by role so posters are prioritized over photos and videos await chain; @@ -1006,6 +1007,7 @@ async function flushOrphanedMedia() { knex('tags_photos').select('media_id'), knex('releases_posters').select('media_id'), knex('releases_photos').select('media_id'), + knex('releases_caps').select('media_id'), knex('releases_covers').select('media_id'), knex('releases_trailers').select('media_id'), knex('releases_teasers').select('media_id'), diff --git a/src/releases.js b/src/releases.js index 1ebf78bf1..4e636b43e 100755 --- a/src/releases.js +++ b/src/releases.js @@ -5,6 +5,7 @@ const inquirer = require('inquirer'); const logger = require('./logger')(__filename); const knex = require('./knex'); const argv = require('./argv'); +const { updateSceneSearch } = require('./update-search'); const { flushOrphanedMedia } = require('./media'); const { graphql } = require('./web/graphql'); @@ -303,6 +304,8 @@ async function deleteScenes(sceneIds) { .whereRaw('id = ANY(:sceneIds)', { sceneIds }) .delete(); + await updateSceneSearch(sceneIds); + logger.info(`Removed ${deleteCount}/${sceneIds.length} scenes`); return deleteCount; diff --git a/src/scrapers/scrapers.js b/src/scrapers/scrapers.js index d9f6b6c7a..b075c73cb 100755 --- a/src/scrapers/scrapers.js +++ b/src/scrapers/scrapers.js @@ -61,6 +61,7 @@ const spizoo = require('./spizoo'); const teamskeet = require('./teamskeet'); const teencoreclub = require('./teencoreclub'); const teenmegaworld = require('./teenmegaworld'); +const tokyohot = require('./tokyohot'); const topwebmodels = require('./topwebmodels'); const traxxx = require('./traxxx'); const vivid = require('./vivid'); @@ -151,6 +152,7 @@ const scrapers = { teencoreclub, teenmegaworld, teamskeet, + tokyohot, topwebmodels, transbella: porndoe, traxxx, @@ -288,6 +290,7 @@ const scrapers = { teencoreclub, teenmegaworld, thatsitcomshow: nubiles, + tokyohot, topwebmodels, transangels: mindgeek, transbella: porndoe, diff --git a/src/scrapers/tokyohot.js b/src/scrapers/tokyohot.js new file mode 100644 index 000000000..b8f1e05d4 --- /dev/null +++ b/src/scrapers/tokyohot.js @@ -0,0 +1,171 @@ +'use strict'; + +const unprint = require('unprint'); + +const slugify = require('../utils/slugify'); + +function scrapeAll(scenes, channel) { + return scenes.map(({ query }) => { + const release = {}; + + const pathname = query.url(); + + release.url = unprint.prefixUrl(pathname, channel.url); + release.entryId = pathname.match(/product\/(\w+)/)?.[1]; + release.shootId = query.attribute('img', 'title'); + + release.title = query.content('.title')?.replace(/^tokyo hot\s*/i, ''); + release.description = query.content('.text'); + + const poster = query.img(); + + release.poster = [ + poster.replace('220x124', '820x462'), + poster, + ]; + + return release; + }); +} + +function scrapeScene({ query }, url, channel) { + const release = {}; + + release.entryId = new URL(url).pathname.match(/product\/(\w+)/)?.[1]; + release.shootId = query.content('//dt[contains(text(), "Product ID")]/following-sibling::dd[1]'); + + release.title = query.content('.contents h2'); + release.description = query.content('.contents .sentence'); + release.date = query.date('//dt[contains(text(), "Release Date")]/following-sibling::dd[1]', 'YYYY/MM/DD'); + release.duration = query.duration('//dt[contains(text(), "Duration")]/following-sibling::dd[1]'); + + release.actors = query.all('.info a[href*="/cast"]').map((el) => ({ + name: unprint.query.content(el), + url: unprint.query.url(el, null, { origin: channel.url }), + })); + + release.tags = query.contents('.info a[href*="type=play"]'); + + const poster = query.poster('.movie video'); + + release.poster = [ + poster, + poster.replace('820x462', '220x124'), + ]; + + release.trailer = query.video('.movie source'); + + release.photos = query.imgs('.scap a', { attribute: 'href' }).map((img) => [ + img, + img.replace('640x480_wlimited', '150x150_default'), + ]); + + release.caps = query.imgs('.vcap a', { attribute: 'href' }).map((img) => [ + img, + img.replace('640x480_wlimited', '120x120_default'), + ]); + + return release; +} + +// measurements are specified as a range in centimeters 85 ~ 89cm +function getMeasurement(string, inches = false) { + if (!string) { + return null; + } + + const value = Array.from(string.matchAll(/(\d+(?:\.\d+)?)\s*cm/g)).at(-1)?.[1]; + + if (!value) { + return null; + } + + if (inches) { + return Math.round(Number(value) * 0.393701); + } + + return Number(value); +} + +function scrapeProfile({ query }) { + const profile = {}; + + const keys = query.contents('.info dt'); + const values = query.contents('.info dd'); + + const bio = Object.fromEntries(keys.map((key, index) => [slugify(key, '_'), values[index]])); + + profile.birthPlace = bio.home_town; + + profile.height = getMeasurement(bio.height); + + profile.cup = bio.cup_size?.replace('cup', '').trim(); + profile.bust = getMeasurement(bio.bust_size, true); + profile.waist = getMeasurement(bio.waist_size, true); + profile.hip = getMeasurement(bio.hip_size || bio.hip, true); + + profile.hairStyle = bio.hair_style; + profile.shoeSize = getMeasurement(bio.shoes_size); + + profile.bloodType = bio.blood_type.replace('type', '').trim(); + + profile.avatar = query.img('#profile img'); + + return profile; +} + +async function fetchLatest(channel, page) { + const url = `${channel.url}/product/?vendor=Tokyo-Hot&page=${page}&order=published_at`; + + const res = await unprint.get(url, { + selectAll: '#main .list .detail', + agent: { + rejectUnauthorized: false, + }, + }); + + if (res.ok) { + return scrapeAll(res.context, channel); + } + + return res.status; +} + +async function fetchScene(url, channel) { + const res = await unprint.get(url, { + agent: { + rejectUnauthorized: false, + }, + }); + + if (res.ok) { + return scrapeScene(res.context, url, channel); + } + + return res.status; +} + +async function fetchProfile(actor, context) { + if (!actor.url) { + // search is cumbersome + return null; + } + + const res = await unprint.get(actor.url, { + agent: { + rejectUnauthorized: false, + }, + }); + + if (res.ok) { + return scrapeProfile(res.context, context); + } + + return res.status; +} + +module.exports = { + fetchLatest, + fetchScene, + fetchProfile, +}; diff --git a/src/store-releases.js b/src/store-releases.js index 44dabc540..8697eb85e 100755 --- a/src/store-releases.js +++ b/src/store-releases.js @@ -16,6 +16,7 @@ const { associateActors, associateDirectors, scrapeActors, toBaseActors } = requ const { associateReleaseTags } = require('./tags'); const { curateEntity } = require('./entities'); const { associateReleaseMedia } = require('./media'); +const { updateSceneSearch, updateMovieSearch } = require('./update-search'); const { notify } = require('./alerts'); async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') { @@ -229,50 +230,6 @@ async function filterDuplicateReleases(releases) { }; } -async function updateSceneSearch(releaseIds) { - logger.info(`Updating search documents for ${releaseIds ? releaseIds.length : 'all' } releases`); - - const documents = await knex.raw(` - SELECT - releases.id AS release_id, - TO_TSVECTOR( - 'english', - COALESCE(releases.title, '') || ' ' || - releases.entry_id || ' ' || - entities.name || ' ' || - entities.slug || ' ' || - COALESCE(array_to_string(entities.alias, ' '), '') || ' ' || - COALESCE(parents.name, '') || ' ' || - COALESCE(parents.slug, '') || ' ' || - COALESCE(array_to_string(parents.alias, ' '), '') || ' ' || - COALESCE(releases.shoot_id, '') || ' ' || - COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' || - STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' || - STRING_AGG(COALESCE(directors.name, ''), ' ') || ' ' || - STRING_AGG(COALESCE(tags.name, ''), ' ') || ' ' || - STRING_AGG(COALESCE(tags_aliases.name, ''), ' ') - ) as document - FROM releases - LEFT JOIN entities ON releases.entity_id = entities.id - LEFT JOIN entities AS parents ON parents.id = entities.parent_id - LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id - LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id - LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id - LEFT JOIN actors ON local_actors.actor_id = actors.id - LEFT JOIN actors AS directors ON local_directors.director_id = directors.id - LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6 - LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true - ${releaseIds ? 'WHERE releases.id = ANY(?)' : ''} - GROUP BY releases.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias; - `, releaseIds && [releaseIds]); - - if (documents.rows?.length > 0) { - await bulkInsert('releases_search', documents.rows, ['release_id']); - } - - await knex.raw('REFRESH MATERIALIZED VIEW releases_summaries;'); -} - async function storeChapters(releases) { const chapters = releases .map((release) => release.chapters?.map((chapter, index) => ({ @@ -380,44 +337,6 @@ async function associateSerieScenes(series, serieScenes) { await bulkInsert('series_scenes', associations, false); } -async function updateMovieSearch(movieIds, target = 'movie') { - logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } ${target}s`); - - const documents = await knex.raw(` - SELECT - ${target}s.id AS ${target}_id, - TO_TSVECTOR( - 'english', - COALESCE(${target}s.title, '') || ' ' || - entities.name || ' ' || - entities.slug || ' ' || - COALESCE(array_to_string(entities.alias, ' '), '') || ' ' || - COALESCE(parents.name, '') || ' ' || - COALESCE(parents.slug, '') || ' ' || - COALESCE(array_to_string(parents.alias, ' '), '') || ' ' || - COALESCE(TO_CHAR(${target}s.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' || - STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' || - STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' || - STRING_AGG(COALESCE(tags.name, ''), ' ') - ) as document - FROM ${target}s - LEFT JOIN entities ON ${target}s.entity_id = entities.id - LEFT JOIN entities AS parents ON parents.id = entities.parent_id - LEFT JOIN ${target}s_scenes ON ${target}s_scenes.${target}_id = ${target}s.id - LEFT JOIN releases ON releases.id = ${target}s_scenes.scene_id - LEFT JOIN releases_actors ON releases_actors.release_id = ${target}s_scenes.scene_id - LEFT JOIN releases_tags ON releases_tags.release_id = releases.id - LEFT JOIN actors ON actors.id = releases_actors.actor_id - LEFT JOIN tags ON tags.id = releases_tags.tag_id - ${movieIds ? `WHERE ${target}s.id = ANY(?)` : ''} - GROUP BY ${target}s.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias; - `, movieIds && [movieIds]); - - if (documents.rows?.length > 0) { - await bulkInsert(`${target}s_search`, documents.rows, [`${target}_id`]); - } -} - async function storeMovies(movies, useBatchId) { if (!movies || movies.length === 0) { return []; diff --git a/src/update-search.js b/src/update-search.js new file mode 100644 index 000000000..bf03dd5d4 --- /dev/null +++ b/src/update-search.js @@ -0,0 +1,92 @@ +'use strict'; + +const knex = require('./knex'); +const logger = require('./logger')(__filename); +const bulkInsert = require('./utils/bulk-insert'); + +async function updateSceneSearch(releaseIds) { + logger.info(`Updating search documents for ${releaseIds ? releaseIds.length : 'all' } releases`); + + const documents = await knex.raw(` + SELECT + releases.id AS release_id, + TO_TSVECTOR( + 'english', + COALESCE(releases.title, '') || ' ' || + releases.entry_id || ' ' || + entities.name || ' ' || + entities.slug || ' ' || + COALESCE(array_to_string(entities.alias, ' '), '') || ' ' || + COALESCE(parents.name, '') || ' ' || + COALESCE(parents.slug, '') || ' ' || + COALESCE(array_to_string(parents.alias, ' '), '') || ' ' || + COALESCE(releases.shoot_id, '') || ' ' || + COALESCE(TO_CHAR(releases.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' || + STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' || + STRING_AGG(COALESCE(directors.name, ''), ' ') || ' ' || + STRING_AGG(COALESCE(tags.name, ''), ' ') || ' ' || + STRING_AGG(COALESCE(tags_aliases.name, ''), ' ') + ) as document + FROM releases + LEFT JOIN entities ON releases.entity_id = entities.id + LEFT JOIN entities AS parents ON parents.id = entities.parent_id + LEFT JOIN releases_actors AS local_actors ON local_actors.release_id = releases.id + LEFT JOIN releases_directors AS local_directors ON local_directors.release_id = releases.id + LEFT JOIN releases_tags AS local_tags ON local_tags.release_id = releases.id + LEFT JOIN actors ON local_actors.actor_id = actors.id + LEFT JOIN actors AS directors ON local_directors.director_id = directors.id + LEFT JOIN tags ON local_tags.tag_id = tags.id AND tags.priority >= 6 + LEFT JOIN tags as tags_aliases ON local_tags.tag_id = tags_aliases.alias_for AND tags_aliases.secondary = true + ${releaseIds ? 'WHERE releases.id = ANY(?)' : ''} + GROUP BY releases.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias; + `, releaseIds && [releaseIds]); + + if (documents.rows?.length > 0) { + await bulkInsert('releases_search', documents.rows, ['release_id']); + } + + await knex.raw('REFRESH MATERIALIZED VIEW releases_summaries;'); +} + +async function updateMovieSearch(movieIds, target = 'movie') { + logger.info(`Updating search documents for ${movieIds ? movieIds.length : 'all' } ${target}s`); + + const documents = await knex.raw(` + SELECT + ${target}s.id AS ${target}_id, + TO_TSVECTOR( + 'english', + COALESCE(${target}s.title, '') || ' ' || + entities.name || ' ' || + entities.slug || ' ' || + COALESCE(array_to_string(entities.alias, ' '), '') || ' ' || + COALESCE(parents.name, '') || ' ' || + COALESCE(parents.slug, '') || ' ' || + COALESCE(array_to_string(parents.alias, ' '), '') || ' ' || + COALESCE(TO_CHAR(${target}s.date, 'YYYY YY MM FMMM FMMonth mon DD FMDD'), '') || ' ' || + STRING_AGG(COALESCE(releases.title, ''), ' ') || ' ' || + STRING_AGG(COALESCE(actors.name, ''), ' ') || ' ' || + STRING_AGG(COALESCE(tags.name, ''), ' ') + ) as document + FROM ${target}s + LEFT JOIN entities ON ${target}s.entity_id = entities.id + LEFT JOIN entities AS parents ON parents.id = entities.parent_id + LEFT JOIN ${target}s_scenes ON ${target}s_scenes.${target}_id = ${target}s.id + LEFT JOIN releases ON releases.id = ${target}s_scenes.scene_id + LEFT JOIN releases_actors ON releases_actors.release_id = ${target}s_scenes.scene_id + LEFT JOIN releases_tags ON releases_tags.release_id = releases.id + LEFT JOIN actors ON actors.id = releases_actors.actor_id + LEFT JOIN tags ON tags.id = releases_tags.tag_id + ${movieIds ? `WHERE ${target}s.id = ANY(?)` : ''} + GROUP BY ${target}s.id, entities.name, entities.slug, entities.alias, parents.name, parents.slug, parents.alias; + `, movieIds && [movieIds]); + + if (documents.rows?.length > 0) { + await bulkInsert(`${target}s_search`, documents.rows, [`${target}_id`]); + } +} + +module.exports = { + updateSceneSearch, + updateMovieSearch, +};