Merge branch 'experimental' into master

This commit is contained in:
DebaucheryLibrarian 2020-08-20 04:57:49 +02:00
commit 23cf5febec
27 changed files with 471 additions and 52 deletions

View File

@ -55,7 +55,7 @@ To build traxxx, run the following command:
`npm run build`
To generate thumbnails for logos and tag photos, install ImageMagick and run:
To generate thumbnails for new logos and tag photos, install ImageMagick and run:
`npm run logos-thumbs`

View File

@ -105,18 +105,20 @@ function sfw() {
}
function photos() {
const photosWithChapterPosters = (this.release.photos || []).concat(this.release.chapters ? this.release.chapters.map(chapter => chapter.poster) : []);
if (this.release.trailer || this.release.teaser) {
// poster will be on trailer video
return this.release.photos;
return photosWithChapterPosters;
}
if (this.release.poster) {
// no trailer, add poster to photos
return [this.release.poster].concat(this.release.photos);
return [this.release.poster].concat(this.release.photos).concat(photosWithChapterPosters);
}
// no poster available
return this.release.photos;
return photosWithChapterPosters;
}
export default {

View File

@ -53,6 +53,7 @@ export default {
display: grid;
grid-template-columns: repeat(auto-fill, minmax(25rem, 1fr));
grid-gap: 1rem;
padding: 1rem;
}
@media(max-width: $breakpoint) {

View File

@ -99,6 +99,51 @@
<p class="description">{{ release.description }}</p>
</div>
<ul
v-if="release.chapters && release.chapters.length > 0"
class="chapters row nolist"
>
<span class="row-label">Chapters</span>
<li
v-for="chapter in release.chapters"
:key="`chapter-${chapter.id}`"
class="chapter"
>
<a
v-if="chapter.poster"
:href="`/media/${chapter.poster.path}`"
target="_blank"
rel="noopener noreferrer"
class="chapter-poster-link"
>
<img
:src="`/media/${chapter.poster.thumbnail}`"
class="chapter-poster"
>
</a>
<div class="chapter-info">
<div class="chapter-header">
<h3 class="chapter-title">{{ chapter.title }}</h3>
<span
v-if="chapter.duration"
class="chapter-duration"
>{{ chapter.duration }}</span>
</div>
<p
v-if="chapter.description"
class="chapter-description"
>{{ chapter.description }}</p>
<ul class="nolist"><li
v-for="tag in chapter.tags"
:key="`chapter-tag-${tag.id}`"
>{{ tag.name }}</li></ul>
</div>
</li>
</ul>
<div class="row row-tidbits">
<div
v-if="release.duration"
@ -143,6 +188,32 @@
<span class="row-label">Shoot date</span>
{{ formatDate(release.productionDate, 'MMMM D, YYYY') }}
</div>
<div
v-if="release.productionLocation"
class="row-tidbit"
>
<span class="row-label">Location</span>
<span class="location">
<span
v-if="release.productionLocation.city"
class="location-segment"
>{{ release.productionLocation.city }}, </span>
<span
v-if="release.productionLocation.state"
class="location-segment"
>{{ release.productionLocation.state }}, </span>
<span
v-if="release.productionLocation.country"
class="location-segment"
>{{ release.productionLocation.country.alias || release.productionLocation.country.name }}
<img
class="flag"
:src="`/img/flags/${release.productionLocation.country.alpha2.toLowerCase()}.svg`"
>
</span>
</span>
</div>
</div>
<div
@ -324,6 +395,41 @@ export default {
text-overflow: ellipsis;
}
.chapter {
display: flex;
background: var(--background);
box-shadow: 0 0 3px var(--shadow-weak);
margin: 0 0 .5rem 0;
}
.chapter-poster {
width: 12rem;
height: 100%;
object-fit: cover;
object-position: center;
}
.chapter-info {
flex-grow: 1;
padding: 1rem 1rem .5rem 1rem;
}
.chapter-header {
display: flex;
justify-content: space-between;
}
.chapter-title {
display: inline-block;
padding: 0;
margin: 0;
}
.flag {
height: 1rem;
margin: 0 0 -.15rem .1rem;
}
.link {
display: inline-flex;
color: var(--link);

View File

@ -69,6 +69,7 @@ function curateRelease(release) {
if (release.scenes) curatedRelease.scenes = release.scenes.map(({ scene }) => curateRelease(scene));
if (release.movies) curatedRelease.movies = release.movies.map(({ movie }) => curateRelease(movie));
if (release.chapters) curatedRelease.chapters = release.chapters.map(chapter => curateRelease(chapter));
if (release.photos) curatedRelease.photos = release.photos.map(({ media }) => media);
if (release.covers) curatedRelease.covers = release.covers.map(({ media }) => media);
if (release.trailer) curatedRelease.trailer = release.trailer.media;
@ -77,6 +78,15 @@ function curateRelease(release) {
if (release.movieTags && release.movieTags.length > 0) curatedRelease.tags = release.movieTags.map(({ tag }) => tag);
if (release.movieActors && release.movieActors.length > 0) curatedRelease.actors = release.movieActors.map(({ actor }) => curateActor(actor, curatedRelease));
if (release.productionLocation) {
curatedRelease.productionLocation = {
raw: release.productionLocation,
city: release.productionCity,
state: release.productionState,
country: release.productionCountry,
};
}
return curatedRelease;
}

View File

@ -237,6 +237,14 @@ const releaseFragment = `
createdAt
shootId
productionDate
productionLocation
productionCity
productionState
productionCountry: countryByProductionCountryAlpha2 {
alpha2
name
alias
}
comment
url
${releaseActorsFragment}
@ -247,6 +255,35 @@ const releaseFragment = `
${releaseTrailerFragment}
${releaseTeaserFragment}
${siteFragment}
chapters {
id
title
description
duration
tags: chaptersTags {
tag {
id
name
slug
}
}
poster: chaptersPosterByChapterId {
media {
index
path
thumbnail
lazy
comment
sfw: sfwMedia {
id
thumbnail
lazy
path
comment
}
}
}
}
studio {
id
name
@ -258,7 +295,7 @@ const releaseFragment = `
id
title
slug
covers: moviesCoversByReleaseId {
covers: moviesCovers {
media {
index
path

View File

@ -7,8 +7,6 @@ function initReleasesActions(store, _router) {
async function fetchReleases({ _commit }, { limit = 10, pageNumber = 1, range = 'latest' }) {
const { before, after, orderBy } = getDateRange(range);
console.log(after, before, orderBy);
const { connection: { releases, totalCount } } = await graphql(`
query Releases(
$limit:Int = 1000,
@ -89,7 +87,7 @@ function initReleasesActions(store, _router) {
type
}
}
covers: moviesCoversByReleaseId {
covers: moviesCovers {
media {
id
path
@ -139,14 +137,14 @@ function initReleasesActions(store, _router) {
lazy
}
}
covers: moviesCoversByReleaseId {
covers: moviesCovers {
media {
id
path
thumbnail
}
}
trailer: moviesTrailerByReleaseId {
trailer: moviesTrailerByMovieId {
media {
id
path

View File

@ -26,8 +26,6 @@ module.exports = {
'amberathome',
'marycarey',
'racqueldevonshire',
// boobpedia
'boobpedia',
// blowpass
'sunlustxxx',
// ddfnetwork

View File

@ -614,6 +614,7 @@ exports.up = knex => Promise.resolve()
table.text('shoot_id');
table.text('entry_id');
table.unique(['entity_id', 'entry_id']);
table.text('url', 1000);
@ -625,6 +626,13 @@ exports.up = knex => Promise.resolve()
table.date('production_date');
table.text('production_location');
table.text('production_city');
table.text('production_state');
table.text('production_country_alpha2', 2)
.references('alpha2')
.inTable('countries');
table.enum('date_precision', ['year', 'month', 'day', 'hour', 'minute', 'second'])
.defaultTo('day');
@ -821,7 +829,7 @@ exports.up = knex => Promise.resolve()
.defaultTo(knex.fn.now());
}))
.then(() => knex.schema.createTable('movies_covers', (table) => {
table.integer('release_id', 16)
table.integer('movie_id', 16)
.notNullable()
.references('id')
.inTable('movies');
@ -831,10 +839,10 @@ exports.up = knex => Promise.resolve()
.references('id')
.inTable('media');
table.unique(['release_id', 'media_id']);
table.unique(['movie_id', 'media_id']);
}))
.then(() => knex.schema.createTable('movies_trailers', (table) => {
table.integer('release_id', 16)
table.integer('movie_id', 16)
.unique()
.notNullable()
.references('id')
@ -845,6 +853,74 @@ exports.up = knex => Promise.resolve()
.references('id')
.inTable('media');
}))
.then(() => knex.schema.createTable('chapters', (table) => {
table.increments('id', 16);
table.integer('release_id', 12)
.references('id')
.inTable('releases')
.notNullable();
table.integer('chapter', 6);
table.unique(['release_id', 'chapter']);
table.text('title');
table.text('description');
table.integer('duration')
.unsigned();
table.integer('created_batch_id', 12)
.references('id')
.inTable('batches');
table.integer('updated_batch_id', 12)
.references('id')
.inTable('batches');
table.datetime('created_at')
.defaultTo(knex.fn.now());
}))
.then(() => knex.schema.createTable('chapters_posters', (table) => {
table.integer('chapter_id', 16)
.notNullable()
.references('id')
.inTable('chapters');
table.text('media_id', 21)
.notNullable()
.references('id')
.inTable('media');
table.unique('chapter_id');
}))
.then(() => knex.schema.createTable('chapters_photos', (table) => {
table.integer('chapter_id', 16)
.notNullable()
.references('id')
.inTable('chapters');
table.text('media_id', 21)
.notNullable()
.references('id')
.inTable('media');
table.unique(['chapter_id', 'media_id']);
}))
.then(() => knex.schema.createTable('chapters_tags', (table) => {
table.integer('tag_id', 12)
.notNullable()
.references('id')
.inTable('tags');
table.integer('chapter_id', 16)
.notNullable()
.references('id')
.inTable('chapters');
table.unique(['tag_id', 'chapter_id']);
}))
// SEARCH
.then(() => { // eslint-disable-line arrow-body-style
// allow vim fold
@ -1024,6 +1100,10 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
DROP TABLE IF EXISTS movies_scenes CASCADE;
DROP TABLE IF EXISTS movies_trailers CASCADE;
DROP TABLE IF EXISTS chapters_tags CASCADE;
DROP TABLE IF EXISTS chapters_posters CASCADE;
DROP TABLE IF EXISTS chapters_photos CASCADE;
DROP TABLE IF EXISTS batches CASCADE;
DROP TABLE IF EXISTS actors_avatars CASCADE;
@ -1042,6 +1122,7 @@ exports.down = (knex) => { // eslint-disable-line arrow-body-style
DROP TABLE IF EXISTS tags_posters CASCADE;
DROP TABLE IF EXISTS tags_photos CASCADE;
DROP TABLE IF EXISTS movies CASCADE;
DROP TABLE IF EXISTS chapters CASCADE;
DROP TABLE IF EXISTS releases CASCADE;
DROP TABLE IF EXISTS actors CASCADE;
DROP TABLE IF EXISTS directors CASCADE;

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 23 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 23 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 20 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 20 KiB

View File

@ -57,6 +57,11 @@ const groups = [
];
const tags = [
{
name: '3d',
slug: '3d',
description: 'Available in 3D.',
},
{
name: '4K',
slug: '4k',

View File

@ -2645,6 +2645,12 @@ const sites = [
accFilter: true,
},
},
// IN THE CRACK
{
slug: 'inthecrack',
name: 'InTheCrack',
url: 'https://inthecrack.com/',
},
// INTERRACIAL PASS
{
slug: '2bigtobetrue',

View File

@ -608,7 +608,7 @@ async function scrapeActors(argNames) {
logger.info(`Scraping profiles for ${actorNames.length} actors`);
const sources = argv.sources || config.profiles || Object.keys(scrapers.actors);
const sources = argv.actorsSources || config.profiles || Object.keys(scrapers.actors);
const entitySlugs = sources.flat();
const [entities, existingActorEntries] = await Promise.all([

View File

@ -617,7 +617,7 @@ async function storeMedias(baseMedias) {
return [...newMediaWithEntries, ...existingHashMedias];
}
async function associateReleaseMedia(releases, type = 'releases') {
async function associateReleaseMedia(releases, type = 'release') {
if (!argv.media) {
return;
}
@ -664,7 +664,7 @@ async function associateReleaseMedia(releases, type = 'releases') {
if (media) {
acc.push({
release_id: releaseId,
[`${type}_id`]: releaseId,
media_id: media.use || media.entry.id,
});
}
@ -675,7 +675,7 @@ async function associateReleaseMedia(releases, type = 'releases') {
.filter(Boolean);
if (associations.length > 0) {
await bulkInsert(`${type}_${role}`, associations, false);
await bulkInsert(`${type}s_${role}`, associations, false);
}
}, Promise.resolve());
}

124
src/scrapers/inthecrack.js Normal file
View File

@ -0,0 +1,124 @@
'use strict';
const moment = require('moment');
const qu = require('../utils/q');
const slugify = require('../utils/slugify');
function scrapeAll(scenes, channel) {
return scenes.map(({ query }) => {
const release = {};
release.url = query.url('a', 'href', { origin: channel.url });
release.entryId = new URL(release.url).pathname.match(/\/Collection\/(\d+)/)[1];
release.shootId = query.cnt('a span:nth-of-type(1)').match(/^\d+/)?.[0];
release.date = query.date('a span:nth-of-type(2)', 'YYYY-MM-DD');
release.actors = (query.q('a img', 'alt') || query.cnt('a span:nth-of-type(1)'))?.match(/[a-zA-Z]+(\s[A-Za-z]+)*/g);
release.poster = release.shootId
? `https://inthecrack.com/assets/images/posters/collections/${release.shootId}.jpg`
: query.img('a img', 'src', { origin: channel.url });
return release;
});
}
function scrapeScene({ query, html }, url, channel) {
const release = {};
release.entryId = new URL(url).pathname.match(/\/Collection\/(\d+)/)[1];
release.shootId = query.cnt('h2 span').match(/^\d+/)?.[0];
release.actors = query.cnt('h2 span')?.match(/[a-zA-Z]+(\s[A-Za-z]+)*/g);
release.description = query.cnt('p#CollectionDescription');
release.productionLocation = query.cnt('.modelCollectionHeader p')?.match(/Shoot Location: (.*)/)?.[1];
release.poster = qu.prefixUrl(html.match(/background-image: url\('(.*)'\)/)?.[1], channel.url);
release.chapters = query.all('.ClipOuter').map((el) => {
const chapter = {};
chapter.title = query.text(el, 'h4');
chapter.description = query.cnt(el, 'p');
chapter.duration = query.dur(el, '.InlineDuration');
const posterStyle = query.style(el, '.clipImage', 'background-image');
const poster = qu.prefixUrl(posterStyle.match(/url\((.*)\)/)?.[1], channel.url);
if (poster) {
const { origin, pathname } = new URL(poster);
chapter.poster = [
`${origin}${pathname}`, // full size
poster,
];
}
if (query.exists(el, '.ThreeDInfo')) {
chapter.tags = ['3d'];
}
return chapter;
});
return release;
}
function scrapeProfile({ query, el }, actorName, entity, include) {
const profile = {};
profile.description = query.cnt('.bio-text');
profile.birthPlace = query.cnt('.birth-place span');
profile.avatar = query.img('.actor-photo img');
if (include.releases) {
return scrapeAll(qu.initAll(el, '.scene'));
}
console.log(profile);
return profile;
}
async function fetchLatest(channel, page = 1) {
const year = moment().subtract(page - 1, ' year').year();
const url = `${channel.url}/Collections/Date/${year}`;
const res = await qu.getAll(url, '.collectionGridLayout li');
if (res.ok) {
return scrapeAll(res.items, channel);
}
return res.status;
}
async function fetchScene(url, channel) {
const res = await qu.get(url);
if (res.ok) {
return scrapeScene(res.item, url, channel);
}
return res.status;
}
async function fetchProfile({ name: actorName }, entity, include) {
const url = `${entity.url}/actors/${slugify(actorName, '_')}`;
const res = await qu.get(url);
if (res.ok) {
return scrapeProfile(res.item, actorName, entity, include);
}
return res.status;
}
module.exports = {
fetchLatest,
fetchScene,
// fetchProfile,
};

View File

@ -27,6 +27,7 @@ const hitzefrei = require('./hitzefrei');
const hush = require('./hush');
const iconmale = require('./iconmale');
const insex = require('./insex');
const inthecrack = require('./inthecrack');
const jayrock = require('./jayrock');
const jesseloadsmonsterfacials = require('./jesseloadsmonsterfacials');
const julesjordan = require('./julesjordan');
@ -108,6 +109,7 @@ module.exports = {
hushpass: hush,
insex,
interracialpass: hush,
inthecrack,
jayrock,
jesseloadsmonsterfacials,
julesjordan,

View File

@ -7,13 +7,14 @@ const logger = require('./logger')(__filename);
const knex = require('./knex');
const slugify = require('./utils/slugify');
const bulkInsert = require('./utils/bulk-insert');
const resolvePlace = require('./utils/resolve-place');
const { formatDate } = require('./utils/qu');
const { associateActors, scrapeActors } = require('./actors');
const { associateReleaseTags } = require('./tags');
const { curateEntity } = require('./entities');
const { associateReleaseMedia } = require('./media');
function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
async function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
const slugBase = release.title
|| (release.actors?.length && `${release.entity.slug} ${release.actors.map(actor => actor.name).join(' ')}`)
|| (release.date && `${release.entity.slug} ${formatDate(release.date, 'YYYY MM DD')}`)
@ -50,6 +51,20 @@ function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
curatedRelease.duration = release.duration;
}
if (release.productionLocation) {
curatedRelease.production_location = release.productionLocation;
if (argv.resolvePlace) {
const productionLocation = await resolvePlace(release.productionLocation);
if (productionLocation) {
curatedRelease.production_city = productionLocation.city;
curatedRelease.production_state = productionLocation.state;
curatedRelease.production_country_alpha2 = productionLocation.country;
}
}
}
if (!existingRelease && !release.id) {
curatedRelease.created_batch_id = batchId;
}
@ -228,6 +243,46 @@ async function updateReleasesSearch(releaseIds) {
}
}
async function storeChapters(releases) {
const chapters = releases.map(release => release.chapters?.map((chapter, index) => ({
title: chapter.title,
description: chapter.description,
releaseId: release.id,
chapter: index + 1,
duration: chapter.duration,
poster: chapter.poster,
photos: chapter.photos,
tags: chapter.tags,
}))).flat().filter(Boolean);
const curatedChapterEntries = chapters.map(chapter => ({
title: chapter.title,
description: chapter.description,
duration: chapter.duration,
release_id: chapter.releaseId,
chapter: chapter.chapter,
}));
const storedChapters = await bulkInsert('chapters', curatedChapterEntries);
const chapterIdsByReleaseIdAndChapter = storedChapters.reduce((acc, chapter) => ({
...acc,
[chapter.release_id]: {
...acc[chapter.release_id],
[chapter.chapter]: chapter.id,
},
}), {});
const chaptersWithId = chapters.map(chapter => ({
...chapter,
id: chapterIdsByReleaseIdAndChapter[chapter.releaseId][chapter.chapter],
}));
await associateReleaseTags(chaptersWithId, 'chapter');
// media is more error-prone, associate separately
await associateReleaseMedia(chaptersWithId, 'chapter');
}
async function storeScenes(releases) {
if (releases.length === 0) {
return [];
@ -241,7 +296,7 @@ async function storeScenes(releases) {
// uniqueness is entity ID + entry ID, filter uniques after adding entities
const { uniqueReleases, duplicateReleases, duplicateReleaseEntries } = await filterDuplicateReleases(releasesWithStudios);
const curatedNewReleaseEntries = uniqueReleases.map(release => curateReleaseEntry(release, batchId));
const curatedNewReleaseEntries = await Promise.all(uniqueReleases.map(release => curateReleaseEntry(release, batchId)));
const storedReleases = await bulkInsert('releases', curatedNewReleaseEntries);
// TODO: update duplicate releases
@ -263,6 +318,8 @@ async function storeScenes(releases) {
await scrapeActors(actors.map(actor => actor.name));
}
await storeChapters(releasesWithId);
logger.info(`Stored ${storedReleaseEntries.length} releases`);
return releasesWithId;
@ -303,13 +360,13 @@ async function storeMovies(movies, movieScenes) {
const { uniqueReleases } = await filterDuplicateReleases(movies);
const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
const curatedMovieEntries = uniqueReleases.map(release => curateReleaseEntry(release, batchId, null, 'movie'));
const curatedMovieEntries = await Promise.all(uniqueReleases.map(release => curateReleaseEntry(release, batchId, null, 'movie')));
const storedMovies = await bulkInsert('movies', curatedMovieEntries, ['entity_id', 'entry_id'], true);
const moviesWithId = attachReleaseIds(movies, storedMovies);
await associateMovieScenes(moviesWithId, movieScenes);
await associateReleaseMedia(moviesWithId, 'movies');
await associateReleaseMedia(moviesWithId, 'movie');
return storedMovies;
}

View File

@ -2,6 +2,7 @@
const knex = require('./knex');
const slugify = require('./utils/slugify');
const bulkInsert = require('./utils/bulk-insert');
async function matchReleaseTags(releases) {
const rawTags = releases
@ -28,7 +29,7 @@ async function matchReleaseTags(releases) {
}
async function getEntityTags(releases) {
const entityIds = releases.map(release => release.entity.id);
const entityIds = releases.map(release => release.entity?.id).filter(Boolean);
const entityTags = await knex('entities_tags').whereIn('entity_id', entityIds);
const entityTagIdsByEntityId = entityTags.reduce((acc, entityTag) => {
@ -44,10 +45,10 @@ async function getEntityTags(releases) {
return entityTagIdsByEntityId;
}
function buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntityId) {
function buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntityId, type) {
const tagAssociations = releases
.map((release) => {
const entityTagIds = entityTagIdsByEntityId[release.entity.id];
const entityTagIds = entityTagIdsByEntityId[release.entity?.id] || [];
const releaseTags = release.tags || [];
const releaseTagIds = releaseTags.every(tag => typeof tag === 'number')
@ -61,7 +62,7 @@ function buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntit
.filter(Boolean),
)]
.map(tagId => ({
release_id: release.id,
[`${type}_id`]: release.id,
tag_id: tagId,
}));
@ -72,34 +73,13 @@ function buildReleaseTagAssociations(releases, tagIdsBySlug, entityTagIdsByEntit
return tagAssociations;
}
async function filterUniqueAssociations(tagAssociations) {
const duplicateAssociations = await knex('releases_tags')
.whereIn(['release_id', 'tag_id'], tagAssociations.map(association => [association.release_id, association.tag_id]));
const duplicateAssociationsByReleaseIdAndTagId = duplicateAssociations.reduce((acc, association) => {
if (!acc[association.release_id]) {
acc[association.release_id] = {};
}
acc[association.release_id][association.tag_id] = true;
return acc;
}, {});
const uniqueAssociations = tagAssociations
.filter(association => !duplicateAssociationsByReleaseIdAndTagId[association.release_id]?.[association.tag_id]);
return uniqueAssociations;
}
async function associateReleaseTags(releases) {
async function associateReleaseTags(releases, type = 'release') {
const tagIdsBySlug = await matchReleaseTags(releases);
const EntityTagIdsByEntityId = await getEntityTags(releases);
const tagAssociations = buildReleaseTagAssociations(releases, tagIdsBySlug, EntityTagIdsByEntityId);
const uniqueAssociations = await filterUniqueAssociations(tagAssociations);
const tagAssociations = buildReleaseTagAssociations(releases, tagIdsBySlug, EntityTagIdsByEntityId, type);
await knex('releases_tags').insert(uniqueAssociations);
await bulkInsert(`${type}s_tags`, tagAssociations, false);
}
module.exports = {

View File

@ -4,6 +4,10 @@ const knex = require('../knex');
const chunk = require('./chunk');
async function bulkUpsert(table, items, conflict, update = true, chunkSize) {
if (items.length === 0) {
return [];
}
const updated = (conflict === false && ':query ON CONFLICT DO NOTHING RETURNING *;')
|| (conflict && update && `
:query ON CONFLICT (${conflict})

View File

@ -20,7 +20,15 @@ async function resolvePlace(query) {
const rawPlace = item.address;
const place = {};
if (rawPlace.city) place.city = rawPlace.city;
if (item.class === 'place' || item.class === 'boundary') {
const location = rawPlace[item.type] || rawPlace.city || rawPlace.place;
if (location) {
place.place = location;
place.city = rawPlace.city || location;
}
}
if (rawPlace.state) place.state = rawPlace.state;
if (rawPlace.country_code) place.country = rawPlace.country_code.toUpperCase();
if (rawPlace.continent) place.continent = rawPlace.continent;