diff --git a/README.md b/README.md index ecfb4caa..537047ec 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ To build traxxx, run the following command: `npm run build` -To generate thumbnails for logos and tag photos, install ImageMagick and run: +To generate thumbnails for new logos and tag photos, install ImageMagick and run: `npm run logos-thumbs` diff --git a/assets/components/releases/media.vue b/assets/components/releases/media.vue index 28ea11f6..249d8eb7 100644 --- a/assets/components/releases/media.vue +++ b/assets/components/releases/media.vue @@ -105,18 +105,20 @@ function sfw() { } function photos() { + const photosWithChapterPosters = (this.release.photos || []).concat(this.release.chapters ? this.release.chapters.map(chapter => chapter.poster) : []); + if (this.release.trailer || this.release.teaser) { // poster will be on trailer video - return this.release.photos; + return photosWithChapterPosters; } if (this.release.poster) { // no trailer, add poster to photos - return [this.release.poster].concat(this.release.photos); + return [this.release.poster].concat(this.release.photos).concat(photosWithChapterPosters); } // no poster available - return this.release.photos; + return photosWithChapterPosters; } export default { diff --git a/assets/components/releases/movies.vue b/assets/components/releases/movies.vue index ca33247b..94284d63 100644 --- a/assets/components/releases/movies.vue +++ b/assets/components/releases/movies.vue @@ -53,6 +53,7 @@ export default { display: grid; grid-template-columns: repeat(auto-fill, minmax(25rem, 1fr)); grid-gap: 1rem; + padding: 1rem; } @media(max-width: $breakpoint) { diff --git a/assets/components/releases/scene.vue b/assets/components/releases/scene.vue index f7681002..4e23ecc5 100644 --- a/assets/components/releases/scene.vue +++ b/assets/components/releases/scene.vue @@ -99,6 +99,51 @@

{{ release.description }}

+ +
Shoot date {{ formatDate(release.productionDate, 'MMMM D, YYYY') }}
+ +
+ Location + + {{ release.productionLocation.city }}, + {{ release.productionLocation.state }}, + {{ release.productionLocation.country.alias || release.productionLocation.country.name }} + + + +
curateRelease(scene)); if (release.movies) curatedRelease.movies = release.movies.map(({ movie }) => curateRelease(movie)); + if (release.chapters) curatedRelease.chapters = release.chapters.map(chapter => curateRelease(chapter)); if (release.photos) curatedRelease.photos = release.photos.map(({ media }) => media); if (release.covers) curatedRelease.covers = release.covers.map(({ media }) => media); if (release.trailer) curatedRelease.trailer = release.trailer.media; @@ -77,6 +78,15 @@ function curateRelease(release) { if (release.movieTags && release.movieTags.length > 0) curatedRelease.tags = release.movieTags.map(({ tag }) => tag); if (release.movieActors && release.movieActors.length > 0) curatedRelease.actors = release.movieActors.map(({ actor }) => curateActor(actor, curatedRelease)); + if (release.productionLocation) { + curatedRelease.productionLocation = { + raw: release.productionLocation, + city: release.productionCity, + state: release.productionState, + country: release.productionCountry, + }; + } + return curatedRelease; } diff --git a/assets/js/fragments.js b/assets/js/fragments.js index f1745723..45f015e4 100644 --- a/assets/js/fragments.js +++ b/assets/js/fragments.js @@ -237,6 +237,14 @@ const releaseFragment = ` createdAt shootId productionDate + productionLocation + productionCity + productionState + productionCountry: countryByProductionCountryAlpha2 { + alpha2 + name + alias + } comment url ${releaseActorsFragment} @@ -247,6 +255,35 @@ const releaseFragment = ` ${releaseTrailerFragment} ${releaseTeaserFragment} ${siteFragment} + chapters { + id + title + description + duration + tags: chaptersTags { + tag { + id + name + slug + } + } + poster: chaptersPosterByChapterId { + media { + index + path + thumbnail + lazy + comment + sfw: sfwMedia { + id + thumbnail + lazy + path + comment + } + } + } + } studio { id name @@ -258,7 +295,7 @@ const releaseFragment = ` id title slug - covers: moviesCoversByReleaseId { + covers: moviesCovers { media { index path diff --git a/assets/js/releases/actions.js b/assets/js/releases/actions.js index 6a9d63e6..a5e5ed32 100644 --- a/assets/js/releases/actions.js +++ b/assets/js/releases/actions.js @@ -7,8 +7,6 @@ function initReleasesActions(store, _router) { async function fetchReleases({ _commit }, { limit = 10, pageNumber = 1, range = 'latest' }) { const { before, after, orderBy } = getDateRange(range); - console.log(after, before, orderBy); - const { connection: { releases, totalCount } } = await graphql(` query Releases( $limit:Int = 1000, @@ -89,7 +87,7 @@ function initReleasesActions(store, _router) { type } } - covers: moviesCoversByReleaseId { + covers: moviesCovers { media { id path @@ -139,14 +137,14 @@ function initReleasesActions(store, _router) { lazy } } - covers: moviesCoversByReleaseId { + covers: moviesCovers { media { id path thumbnail } } - trailer: moviesTrailerByReleaseId { + trailer: moviesTrailerByMovieId { media { id path diff --git a/config/default.js b/config/default.js index c0abb69d..80978805 100644 --- a/config/default.js +++ b/config/default.js @@ -26,8 +26,6 @@ module.exports = { 'amberathome', 'marycarey', 'racqueldevonshire', - // boobpedia - 'boobpedia', // blowpass 'sunlustxxx', // ddfnetwork diff --git a/migrations/20190325001339_releases.js b/migrations/20190325001339_releases.js index 0d9271f0..2f3d2cb1 100644 --- a/migrations/20190325001339_releases.js +++ b/migrations/20190325001339_releases.js @@ -614,6 +614,7 @@ exports.up = knex => Promise.resolve() table.text('shoot_id'); table.text('entry_id'); + table.unique(['entity_id', 'entry_id']); table.text('url', 1000); @@ -625,6 +626,13 @@ exports.up = knex => Promise.resolve() table.date('production_date'); + table.text('production_location'); + table.text('production_city'); + table.text('production_state'); + table.text('production_country_alpha2', 2) + .references('alpha2') + .inTable('countries'); + table.enum('date_precision', ['year', 'month', 'day', 'hour', 'minute', 'second']) .defaultTo('day'); @@ -821,7 +829,7 @@ exports.up = knex => Promise.resolve() .defaultTo(knex.fn.now()); })) .then(() => knex.schema.createTable('movies_covers', (table) => { - table.integer('release_id', 16) + table.integer('movie_id', 16) .notNullable() .references('id') .inTable('movies'); @@ -831,10 +839,10 @@ exports.up = knex => Promise.resolve() .references('id') .inTable('media'); - table.unique(['release_id', 'media_id']); + table.unique(['movie_id', 'media_id']); })) .then(() => knex.schema.createTable('movies_trailers', (table) => { - table.integer('release_id', 16) + table.integer('movie_id', 16) .unique() .notNullable() .references('id') @@ -845,6 +853,74 @@ exports.up = knex => Promise.resolve() .references('id') .inTable('media'); })) + .then(() => knex.schema.createTable('chapters', (table) => { + table.increments('id', 16); + + table.integer('release_id', 12) + .references('id') + .inTable('releases') + .notNullable(); + + table.integer('chapter', 6); + + table.unique(['release_id', 'chapter']); + + table.text('title'); + table.text('description'); + + table.integer('duration') + .unsigned(); + + table.integer('created_batch_id', 12) + .references('id') + .inTable('batches'); + + table.integer('updated_batch_id', 12) + .references('id') + .inTable('batches'); + + table.datetime('created_at') + .defaultTo(knex.fn.now()); + })) + .then(() => knex.schema.createTable('chapters_posters', (table) => { + table.integer('chapter_id', 16) + .notNullable() + .references('id') + .inTable('chapters'); + + table.text('media_id', 21) + .notNullable() + .references('id') + .inTable('media'); + + table.unique('chapter_id'); + })) + .then(() => knex.schema.createTable('chapters_photos', (table) => { + table.integer('chapter_id', 16) + .notNullable() + .references('id') + .inTable('chapters'); + + table.text('media_id', 21) + .notNullable() + .references('id') + .inTable('media'); + + table.unique(['chapter_id', 'media_id']); + })) + .then(() => knex.schema.createTable('chapters_tags', (table) => { + table.integer('tag_id', 12) + .notNullable() + .references('id') + .inTable('tags'); + + table.integer('chapter_id', 16) + .notNullable() + .references('id') + .inTable('chapters'); + + table.unique(['tag_id', 'chapter_id']); + })) // SEARCH .then(() => { // eslint-disable-line arrow-body-style // allow vim fold @@ -1024,6 +1100,10 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style DROP TABLE IF EXISTS movies_scenes CASCADE; DROP TABLE IF EXISTS movies_trailers CASCADE; + DROP TABLE IF EXISTS chapters_tags CASCADE; + DROP TABLE IF EXISTS chapters_posters CASCADE; + DROP TABLE IF EXISTS chapters_photos CASCADE; + DROP TABLE IF EXISTS batches CASCADE; DROP TABLE IF EXISTS actors_avatars CASCADE; @@ -1042,6 +1122,7 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style DROP TABLE IF EXISTS tags_posters CASCADE; DROP TABLE IF EXISTS tags_photos CASCADE; DROP TABLE IF EXISTS movies CASCADE; + DROP TABLE IF EXISTS chapters CASCADE; DROP TABLE IF EXISTS releases CASCADE; DROP TABLE IF EXISTS actors CASCADE; DROP TABLE IF EXISTS directors CASCADE; diff --git a/public/img/logos/inthecrack/favicon.png b/public/img/logos/inthecrack/favicon.png new file mode 100644 index 00000000..8e5c2dcc Binary files /dev/null and b/public/img/logos/inthecrack/favicon.png differ diff --git a/public/img/logos/inthecrack/inthecrack.png b/public/img/logos/inthecrack/inthecrack.png new file mode 100644 index 00000000..a233ab42 Binary files /dev/null and b/public/img/logos/inthecrack/inthecrack.png differ diff --git a/public/img/logos/inthecrack/lazy/inthecrack.png b/public/img/logos/inthecrack/lazy/inthecrack.png new file mode 100644 index 00000000..6d0529af Binary files /dev/null and b/public/img/logos/inthecrack/lazy/inthecrack.png differ diff --git a/public/img/logos/inthecrack/lazy/network.png b/public/img/logos/inthecrack/lazy/network.png new file mode 100644 index 00000000..74b29a32 Binary files /dev/null and b/public/img/logos/inthecrack/lazy/network.png differ diff --git a/public/img/logos/inthecrack/misc/in-the-crack_original.png b/public/img/logos/inthecrack/misc/in-the-crack_original.png new file mode 100644 index 00000000..cd8e0fee Binary files /dev/null and b/public/img/logos/inthecrack/misc/in-the-crack_original.png differ diff --git a/public/img/logos/inthecrack/network.png b/public/img/logos/inthecrack/network.png new file mode 100644 index 00000000..fe68dba7 Binary files /dev/null and b/public/img/logos/inthecrack/network.png differ diff --git a/public/img/logos/inthecrack/thumbs/inthecrack.png b/public/img/logos/inthecrack/thumbs/inthecrack.png new file mode 100644 index 00000000..23c092b5 Binary files /dev/null and b/public/img/logos/inthecrack/thumbs/inthecrack.png differ diff --git a/public/img/logos/inthecrack/thumbs/network.png b/public/img/logos/inthecrack/thumbs/network.png new file mode 100644 index 00000000..cd78d321 Binary files /dev/null and b/public/img/logos/inthecrack/thumbs/network.png differ diff --git a/seeds/00_tags.js b/seeds/00_tags.js index 68a81d30..cd532887 100644 --- a/seeds/00_tags.js +++ b/seeds/00_tags.js @@ -57,6 +57,11 @@ const groups = [ ]; const tags = [ + { + name: '3d', + slug: '3d', + description: 'Available in 3D.', + }, { name: '4K', slug: '4k', diff --git a/seeds/02_sites.js b/seeds/02_sites.js index 8aeef7ef..f44141b8 100644 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -2645,6 +2645,12 @@ const sites = [ accFilter: true, }, }, + // IN THE CRACK + { + slug: 'inthecrack', + name: 'InTheCrack', + url: 'https://inthecrack.com/', + }, // INTERRACIAL PASS { slug: '2bigtobetrue', diff --git a/src/actors.js b/src/actors.js index 1f825675..b8c4e56c 100644 --- a/src/actors.js +++ b/src/actors.js @@ -608,7 +608,7 @@ async function scrapeActors(argNames) { logger.info(`Scraping profiles for ${actorNames.length} actors`); - const sources = argv.sources || config.profiles || Object.keys(scrapers.actors); + const sources = argv.actorsSources || config.profiles || Object.keys(scrapers.actors); const entitySlugs = sources.flat(); const [entities, existingActorEntries] = await Promise.all([ diff --git a/src/media.js b/src/media.js index f044fbc0..672cc64d 100644 --- a/src/media.js +++ b/src/media.js @@ -617,7 +617,7 @@ async function storeMedias(baseMedias) { return [...newMediaWithEntries, ...existingHashMedias]; } -async function associateReleaseMedia(releases, type = 'releases') { +async function associateReleaseMedia(releases, type = 'release') { if (!argv.media) { return; } @@ -664,7 +664,7 @@ async function associateReleaseMedia(releases, type = 'releases') { if (media) { acc.push({ - release_id: releaseId, + [`${type}_id`]: releaseId, media_id: media.use || media.entry.id, }); } @@ -675,7 +675,7 @@ async function associateReleaseMedia(releases, type = 'releases') { .filter(Boolean); if (associations.length > 0) { - await bulkInsert(`${type}_${role}`, associations, false); + await bulkInsert(`${type}s_${role}`, associations, false); } }, Promise.resolve()); } diff --git a/src/scrapers/inthecrack.js b/src/scrapers/inthecrack.js new file mode 100644 index 00000000..3dad77a3 --- /dev/null +++ b/src/scrapers/inthecrack.js @@ -0,0 +1,124 @@ +'use strict'; + +const moment = require('moment'); + +const qu = require('../utils/q'); +const slugify = require('../utils/slugify'); + +function scrapeAll(scenes, channel) { + return scenes.map(({ query }) => { + const release = {}; + + release.url = query.url('a', 'href', { origin: channel.url }); + release.entryId = new URL(release.url).pathname.match(/\/Collection\/(\d+)/)[1]; + + release.shootId = query.cnt('a span:nth-of-type(1)').match(/^\d+/)?.[0]; + release.date = query.date('a span:nth-of-type(2)', 'YYYY-MM-DD'); + + release.actors = (query.q('a img', 'alt') || query.cnt('a span:nth-of-type(1)'))?.match(/[a-zA-Z]+(\s[A-Za-z]+)*/g); + + release.poster = release.shootId + ? `https://inthecrack.com/assets/images/posters/collections/${release.shootId}.jpg` + : query.img('a img', 'src', { origin: channel.url }); + + return release; + }); +} + +function scrapeScene({ query, html }, url, channel) { + const release = {}; + + release.entryId = new URL(url).pathname.match(/\/Collection\/(\d+)/)[1]; + release.shootId = query.cnt('h2 span').match(/^\d+/)?.[0]; + + release.actors = query.cnt('h2 span')?.match(/[a-zA-Z]+(\s[A-Za-z]+)*/g); + + release.description = query.cnt('p#CollectionDescription'); + release.productionLocation = query.cnt('.modelCollectionHeader p')?.match(/Shoot Location: (.*)/)?.[1]; + + release.poster = qu.prefixUrl(html.match(/background-image: url\('(.*)'\)/)?.[1], channel.url); + + release.chapters = query.all('.ClipOuter').map((el) => { + const chapter = {}; + + chapter.title = query.text(el, 'h4'); + chapter.description = query.cnt(el, 'p'); + chapter.duration = query.dur(el, '.InlineDuration'); + + const posterStyle = query.style(el, '.clipImage', 'background-image'); + const poster = qu.prefixUrl(posterStyle.match(/url\((.*)\)/)?.[1], channel.url); + + if (poster) { + const { origin, pathname } = new URL(poster); + + chapter.poster = [ + `${origin}${pathname}`, // full size + poster, + ]; + } + + if (query.exists(el, '.ThreeDInfo')) { + chapter.tags = ['3d']; + } + + return chapter; + }); + + return release; +} + +function scrapeProfile({ query, el }, actorName, entity, include) { + const profile = {}; + + profile.description = query.cnt('.bio-text'); + profile.birthPlace = query.cnt('.birth-place span'); + + profile.avatar = query.img('.actor-photo img'); + + if (include.releases) { + return scrapeAll(qu.initAll(el, '.scene')); + } + + console.log(profile); + return profile; +} + +async function fetchLatest(channel, page = 1) { + const year = moment().subtract(page - 1, ' year').year(); + + const url = `${channel.url}/Collections/Date/${year}`; + const res = await qu.getAll(url, '.collectionGridLayout li'); + + if (res.ok) { + return scrapeAll(res.items, channel); + } + + return res.status; +} + +async function fetchScene(url, channel) { + const res = await qu.get(url); + + if (res.ok) { + return scrapeScene(res.item, url, channel); + } + + return res.status; +} + +async function fetchProfile({ name: actorName }, entity, include) { + const url = `${entity.url}/actors/${slugify(actorName, '_')}`; + const res = await qu.get(url); + + if (res.ok) { + return scrapeProfile(res.item, actorName, entity, include); + } + + return res.status; +} + +module.exports = { + fetchLatest, + fetchScene, + // fetchProfile, +}; diff --git a/src/scrapers/scrapers.js b/src/scrapers/scrapers.js index ab5ab528..f67add47 100644 --- a/src/scrapers/scrapers.js +++ b/src/scrapers/scrapers.js @@ -27,6 +27,7 @@ const hitzefrei = require('./hitzefrei'); const hush = require('./hush'); const iconmale = require('./iconmale'); const insex = require('./insex'); +const inthecrack = require('./inthecrack'); const jayrock = require('./jayrock'); const jesseloadsmonsterfacials = require('./jesseloadsmonsterfacials'); const julesjordan = require('./julesjordan'); @@ -108,6 +109,7 @@ module.exports = { hushpass: hush, insex, interracialpass: hush, + inthecrack, jayrock, jesseloadsmonsterfacials, julesjordan, diff --git a/src/store-releases.js b/src/store-releases.js index 40386844..1cfd0998 100644 --- a/src/store-releases.js +++ b/src/store-releases.js @@ -7,13 +7,14 @@ const logger = require('./logger')(__filename); const knex = require('./knex'); const slugify = require('./utils/slugify'); const bulkInsert = require('./utils/bulk-insert'); +const resolvePlace = require('./utils/resolve-place'); const { formatDate } = require('./utils/qu'); const { associateActors, scrapeActors } = require('./actors'); const { associateReleaseTags } = require('./tags'); const { curateEntity } = require('./entities'); const { associateReleaseMedia } = require('./media'); -function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') { +async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') { const slugBase = release.title || (release.actors?.length && `${release.entity.slug} ${release.actors.map(actor => actor.name).join(' ')}`) || (release.date && `${release.entity.slug} ${formatDate(release.date, 'YYYY MM DD')}`) @@ -50,6 +51,20 @@ function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') { curatedRelease.duration = release.duration; } + if (release.productionLocation) { + curatedRelease.production_location = release.productionLocation; + + if (argv.resolvePlace) { + const productionLocation = await resolvePlace(release.productionLocation); + + if (productionLocation) { + curatedRelease.production_city = productionLocation.city; + curatedRelease.production_state = productionLocation.state; + curatedRelease.production_country_alpha2 = productionLocation.country; + } + } + } + if (!existingRelease && !release.id) { curatedRelease.created_batch_id = batchId; } @@ -228,6 +243,46 @@ async function updateReleasesSearch(releaseIds) { } } +async function storeChapters(releases) { + const chapters = releases.map(release => release.chapters?.map((chapter, index) => ({ + title: chapter.title, + description: chapter.description, + releaseId: release.id, + chapter: index + 1, + duration: chapter.duration, + poster: chapter.poster, + photos: chapter.photos, + tags: chapter.tags, + }))).flat().filter(Boolean); + + const curatedChapterEntries = chapters.map(chapter => ({ + title: chapter.title, + description: chapter.description, + duration: chapter.duration, + release_id: chapter.releaseId, + chapter: chapter.chapter, + })); + + const storedChapters = await bulkInsert('chapters', curatedChapterEntries); + const chapterIdsByReleaseIdAndChapter = storedChapters.reduce((acc, chapter) => ({ + ...acc, + [chapter.release_id]: { + ...acc[chapter.release_id], + [chapter.chapter]: chapter.id, + }, + }), {}); + + const chaptersWithId = chapters.map(chapter => ({ + ...chapter, + id: chapterIdsByReleaseIdAndChapter[chapter.releaseId][chapter.chapter], + })); + + await associateReleaseTags(chaptersWithId, 'chapter'); + + // media is more error-prone, associate separately + await associateReleaseMedia(chaptersWithId, 'chapter'); +} + async function storeScenes(releases) { if (releases.length === 0) { return []; @@ -241,7 +296,7 @@ async function storeScenes(releases) { // uniqueness is entity ID + entry ID, filter uniques after adding entities const { uniqueReleases, duplicateReleases, duplicateReleaseEntries } = await filterDuplicateReleases(releasesWithStudios); - const curatedNewReleaseEntries = uniqueReleases.map(release => curateReleaseEntry(release, batchId)); + const curatedNewReleaseEntries = await Promise.all(uniqueReleases.map(release => curateReleaseEntry(release, batchId))); const storedReleases = await bulkInsert('releases', curatedNewReleaseEntries); // TODO: update duplicate releases @@ -263,6 +318,8 @@ async function storeScenes(releases) { await scrapeActors(actors.map(actor => actor.name)); } + await storeChapters(releasesWithId); + logger.info(`Stored ${storedReleaseEntries.length} releases`); return releasesWithId; @@ -303,13 +360,13 @@ async function storeMovies(movies, movieScenes) { const { uniqueReleases } = await filterDuplicateReleases(movies); const [batchId] = await knex('batches').insert({ comment: null }).returning('id'); - const curatedMovieEntries = uniqueReleases.map(release => curateReleaseEntry(release, batchId, null, 'movie')); + const curatedMovieEntries = await Promise.all(uniqueReleases.map(release => curateReleaseEntry(release, batchId, null, 'movie'))); const storedMovies = await bulkInsert('movies', curatedMovieEntries, ['entity_id', 'entry_id'], true); const moviesWithId = attachReleaseIds(movies, storedMovies); await associateMovieScenes(moviesWithId, movieScenes); - await associateReleaseMedia(moviesWithId, 'movies'); + await associateReleaseMedia(moviesWithId, 'movie'); return storedMovies; } diff --git a/src/tags.js b/src/tags.js index 2efb56bd..d8cb94af 100644 --- a/src/tags.js +++ b/src/tags.js @@ -2,6 +2,7 @@ const knex = require('./knex'); const slugify = require('./utils/slugify'); +const bulkInsert = require('./utils/bulk-insert'); async function matchReleaseTags(releases) { const rawTags = releases @@ -28,7 +29,7 @@ async function matchReleaseTags(releases) { } async function getEntityTags(releases) { - const entityIds = releases.map(release => release.entity.id); + const entityIds = releases.map(release => release.entity?.id).filter(Boolean); const entityTags = await knex('entities_tags').whereIn('entity_id', entityIds); const entityTagIdsByEntityId = entityTags.reduce((acc, entityTag) => { @@ -44,10 +45,10 @@ async function getEntityTags(releases) { return entityTagIdsByEntityId; } -function buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntityId) { +function buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntityId, type) { const tagAssociations = releases .map((release) => { - const entityTagIds = entityTagIdsByEntityId[release.entity.id]; + const entityTagIds = entityTagIdsByEntityId[release.entity?.id] || []; const releaseTags = release.tags || []; const releaseTagIds = releaseTags.every(tag => typeof tag === 'number') @@ -61,7 +62,7 @@ function buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntit .filter(Boolean), )] .map(tagId => ({ - release_id: release.id, + [`${type}_id`]: release.id, tag_id: tagId, })); @@ -72,34 +73,13 @@ function buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntit return tagAssociations; } -async function filterUniqueAssociations(tagAssociations) { - const duplicateAssociations = await knex('releases_tags') - .whereIn(['release_id', 'tag_id'], tagAssociations.map(association => [association.release_id, association.tag_id])); - - const duplicateAssociationsByReleaseIdAndTagId = duplicateAssociations.reduce((acc, association) => { - if (!acc[association.release_id]) { - acc[association.release_id] = {}; - } - - acc[association.release_id][association.tag_id] = true; - - return acc; - }, {}); - - const uniqueAssociations = tagAssociations - .filter(association => !duplicateAssociationsByReleaseIdAndTagId[association.release_id]?.[association.tag_id]); - - return uniqueAssociations; -} - -async function associateReleaseTags(releases) { +async function associateReleaseTags(releases, type = 'release') { const tagIdsBySlug = await matchReleaseTags(releases); const EntityTagIdsByEntityId = await getEntityTags(releases); - const tagAssociations = buildReleaseTagAssociations(releases, tagIdsBySlug, EntityTagIdsByEntityId); - const uniqueAssociations = await filterUniqueAssociations(tagAssociations); + const tagAssociations = buildReleaseTagAssociations(releases, tagIdsBySlug, EntityTagIdsByEntityId, type); - await knex('releases_tags').insert(uniqueAssociations); + await bulkInsert(`${type}s_tags`, tagAssociations, false); } module.exports = { diff --git a/src/utils/bulk-insert.js b/src/utils/bulk-insert.js index 7ef5fffc..b2af09d7 100644 --- a/src/utils/bulk-insert.js +++ b/src/utils/bulk-insert.js @@ -4,6 +4,10 @@ const knex = require('../knex'); const chunk = require('./chunk'); async function bulkUpsert(table, items, conflict, update = true, chunkSize) { + if (items.length === 0) { + return []; + } + const updated = (conflict === false && ':query ON CONFLICT DO NOTHING RETURNING *;') || (conflict && update && ` :query ON CONFLICT (${conflict}) diff --git a/src/utils/resolve-place.js b/src/utils/resolve-place.js index 4c455995..cea15148 100644 --- a/src/utils/resolve-place.js +++ b/src/utils/resolve-place.js @@ -20,7 +20,15 @@ async function resolvePlace(query) { const rawPlace = item.address; const place = {}; - if (rawPlace.city) place.city = rawPlace.city; + if (item.class === 'place' || item.class === 'boundary') { + const location = rawPlace[item.type] || rawPlace.city || rawPlace.place; + + if (location) { + place.place = location; + place.city = rawPlace.city || location; + } + } + if (rawPlace.state) place.state = rawPlace.state; if (rawPlace.country_code) place.country = rawPlace.country_code.toUpperCase(); if (rawPlace.continent) place.continent = rawPlace.continent;