diff --git a/assets/components/releases/banner.vue b/assets/components/releases/banner.vue index e5a016dd..d21d460e 100644 --- a/assets/components/releases/banner.vue +++ b/assets/components/releases/banner.vue @@ -112,9 +112,9 @@ v-if="!me" class="item-container item-more" >Sign up for more photos, trailers and features! + >Log in for more photos, trailers and features! diff --git a/assets/js/main.js b/assets/js/main.js index b5a0a1ae..caefe8b7 100644 --- a/assets/js/main.js +++ b/assets/js/main.js @@ -125,6 +125,10 @@ async function init() { app.directive('tooltip', { beforeMount(el, binding) { + if (!binding.value) { + return; + } + // don't include HTML in native title attribute const textEl = document.createElement('div'); textEl.innerHTML = binding.value; diff --git a/assets/js/releases/actions.js b/assets/js/releases/actions.js index d2beca7b..7933d5bc 100644 --- a/assets/js/releases/actions.js +++ b/assets/js/releases/actions.js @@ -183,7 +183,7 @@ function initReleasesActions(store, router) { isS3 } } - posters: moviesPosterByMovieId { + poster: moviesPosterByMovieId { media { id path diff --git a/migrations/20190325001339_releases.js b/migrations/20190325001339_releases.js index a6d738e9..be6a8c78 100644 --- a/migrations/20190325001339_releases.js +++ b/migrations/20190325001339_releases.js @@ -667,6 +667,11 @@ exports.up = knex => Promise.resolve() .onDelete('cascade'); table.datetime('created_at') + .notNullable() + .defaultTo(knex.fn.now()); + + table.datetime('updated_at') + .notNullable() .defaultTo(knex.fn.now()); })) .then(() => knex.schema.createTable('releases_actors', (table) => { diff --git a/public/img/logos/kirakira/kirakira.png b/public/img/logos/kirakira/kirakira.png new file mode 100644 index 00000000..3f00a5ab Binary files /dev/null and b/public/img/logos/kirakira/kirakira.png differ diff --git a/public/img/logos/kirakira/misc/kira_kira_crown.png b/public/img/logos/kirakira/misc/kira_kira_crown.png new file mode 100644 index 00000000..f8a691d2 Binary files /dev/null and b/public/img/logos/kirakira/misc/kira_kira_crown.png differ diff --git a/public/img/logos/kirakira/misc/kira_kira_gold.png b/public/img/logos/kirakira/misc/kira_kira_gold.png new file mode 100644 index 00000000..03ac996d Binary files /dev/null and b/public/img/logos/kirakira/misc/kira_kira_gold.png differ diff --git a/public/img/logos/kirakira/misc/kira_kira_serif.png b/public/img/logos/kirakira/misc/kira_kira_serif.png new file mode 100644 index 00000000..081fa3c8 Binary files /dev/null and b/public/img/logos/kirakira/misc/kira_kira_serif.png differ diff --git a/public/img/logos/kirakira/misc/kira_kira_serif_wide.png b/public/img/logos/kirakira/misc/kira_kira_serif_wide.png new file mode 100644 index 00000000..97873632 Binary files /dev/null and b/public/img/logos/kirakira/misc/kira_kira_serif_wide.png differ diff --git a/public/img/logos/kirakira/misc/kira_kira_writing.png b/public/img/logos/kirakira/misc/kira_kira_writing.png new file mode 100644 index 00000000..c89c0ae6 Binary files /dev/null and b/public/img/logos/kirakira/misc/kira_kira_writing.png differ diff --git a/src/scrapers/dorcel.js b/src/scrapers/dorcel.js index a41cfd62..0d0fd334 100644 --- a/src/scrapers/dorcel.js +++ b/src/scrapers/dorcel.js @@ -50,10 +50,15 @@ function scrapeScene({ query }, url, channel) { const fallbackPoster = query.img('.player img'); release.poster = query.sourceSet('.player img', 'data-srcset') || [fallbackPoster.replace('_crop', ''), fallbackPoster]; - release.movie = { - title: query.cnt('.movie a'), - url: query.url('.movie a', 'href', { origin: channel.url }), - }; + const movieUrl = query.url('.movie a', 'href', { origin: channel.url }); + + if (movieUrl) { + release.movie = { + entryId: new URL(movieUrl).pathname.match(/\/porn-movie\/([\w-]+)/)?.[1], + title: query.cnt('.movie a'), + url: query.url('.movie a', 'href', { origin: channel.url }), + }; + } return release; } @@ -92,8 +97,20 @@ function scrapeMovie({ query, el }, url, channel) { avatar: query.sourceSet(actorEl, '.thumbnail img', 'data-srcset'), })); - release.poster = query.sourceSet('.banner', 'data-srcset'); - release.covers = [query.sourceSet('.cover', 'data-srcset')]; + release.poster = query.sourceSet('.banner', 'data-src')?.[0]; + release.covers = [query.all(query.el('.cover').parentElement, 'source') + ?.map(coverEl => query.sourceSet(coverEl, null, 'data-srcset')) + .flat() + .sort((coverA, coverB) => { + const resA = Number(coverA.match(/_(\d{3,})_/)?.[1]); + const resB = Number(coverB.match(/_(\d{3,})_/)?.[1]); + + if (resA < resB) return 1; + if (resA > resB) return -1; + + return 0; + }) + .concat(query.sourceSet('.cover', 'data-src')?.[0])]; release.scenes = scrapeAll(qu.initAll(el, '.scene'), channel); @@ -120,13 +137,25 @@ async function scrapeProfile({ query, el }, entity, avatar) { return profile; } -async function fetchLatest(channel, page = 1) { +async function beforeFetchLatest(channel) { + // scene page only seems to accept language preferences from session + const session = qu.session(); + + await qu.getAll(`${channel.url}/en/news-videos-x-marc-dorcel`, '.scene', { + 'X-Requested-With': 'XMLHttpRequest', + 'Accept-Language': 'en-US,en', // fetch English rather than French titles + }, { session }); + + return session; +} + +async function fetchLatest(channel, page = 1, options, { beforeFetchLatest: session }) { const url = `${channel.url}/scene/list/more/?lang=en&page=${page}&sorting=new`; const res = await qu.getAll(url, '.scene', { 'X-Requested-With': 'XMLHttpRequest', 'Accept-Language': 'en-US,en', // fetch English rather than French titles - }); + }, { session }); if (res.ok) { return scrapeAll(res.items, channel); @@ -152,8 +181,9 @@ async function fetchMovies(channel, page = 1) { } async function fetchScene(url, channel) { - const res = await qu.get(url, '.content', { + const res = await qu.get(url, null, { 'Accept-Language': 'en-US,en', // fetch English rather than French titles + Referer: `${channel.url}/en/news-videos-x-marc-dorcel`, }); if (res.ok) { @@ -166,6 +196,7 @@ async function fetchScene(url, channel) { async function fetchMovie(url, channel) { const res = await qu.get(url, '.content', { 'Accept-Language': 'en-US,en', // fetch English rather than French titles + Referer: `${channel.url}/en/porn-movie`, }); if (res.ok) { @@ -202,6 +233,7 @@ async function fetchProfile(baseActor, { entity }) { } module.exports = { + beforeFetchLatest, fetchLatest, fetchScene, fetchMovie, diff --git a/src/store-releases.js b/src/store-releases.js index 434fe00f..75f909de 100644 --- a/src/store-releases.js +++ b/src/store-releases.js @@ -315,11 +315,33 @@ async function storeScenes(releases) { const { uniqueReleases, duplicateReleases, duplicateReleaseEntries } = await filterDuplicateReleases(releasesWithStudios); const curatedNewReleaseEntries = await Promise.all(uniqueReleases.map(release => curateReleaseEntry(release, batchId))); - const storedReleases = await bulkInsert('releases', curatedNewReleaseEntries); const storedReleaseEntries = Array.isArray(storedReleases) ? storedReleases : []; - const releasesWithId = attachReleaseIds([].concat(uniqueReleases, duplicateReleases), [].concat(storedReleaseEntries, duplicateReleaseEntries)); + + const uniqueReleasesWithId = attachReleaseIds(uniqueReleases, storedReleaseEntries); + const duplicateReleasesWithId = attachReleaseIds(duplicateReleases, duplicateReleaseEntries); + const releasesWithId = uniqueReleasesWithId.concat(duplicateReleasesWithId); + + try { + await knex.raw(` + UPDATE releases + SET url = COALESCE(new.url, releases.url), + date = COALESCE(new.date, releases.date), + title = COALESCE(new.title, releases.title), + description = COALESCE(new.description, releases.description), + duration = COALESCE(new.duration, releases.duration), + deep = new.url IS NOT NULL, + updated_at = NOW() + FROM json_to_recordset(:scenes) + AS new(id int, url text, date timestamptz, title text, description text, duration integer, deep boolean) + WHERE releases.id = new.id; + `, { + scenes: JSON.stringify(duplicateReleasesWithId), + }); + } catch (error) { + console.log(error); + } const [actors] = await Promise.all([ associateActors(releasesWithId, batchId), diff --git a/src/updates.js b/src/updates.js index cfab379c..bcf4f299 100644 --- a/src/updates.js +++ b/src/updates.js @@ -44,7 +44,17 @@ async function filterUniqueReleases(releases) { const duplicateReleaseEntries = await knex('releases') .select(knex.raw('releases.*, row_to_json(entities) as entity')) .leftJoin('entities', 'entities.id', 'releases.entity_id') - .whereIn(['entity_id', 'entry_id'], releaseIdentifiers); + .whereIn(['entity_id', 'entry_id'], releaseIdentifiers) + .where((builder) => { + // check if previously upcoming scenes can be excluded from duplicates to be rescraped for release day updates + builder + .where('deep', true) // scene is already deep scraped + .orWhereNull('date') + .orWhereNotIn('date_precision', ['day', 'minute']) // don't worry about scenes without (accurate) dates for now + .orWhere(knex.raw('NOW() - date > INTERVAL \'12 hours\'')) // scene is still upcoming, with a rough offset to wait for the end of the day west of UTC + .orWhere(knex.raw('updated_at - date > INTERVAL \'1 day\'')); // scene was updated after the release date, no updated expected + }); + const duplicateReleases = duplicateReleaseEntries.map(release => curateRelease(release)); const duplicateReleasesByEntityIdAndEntryId = duplicateReleases.reduce(mapReleasesToEntityIdAndEntryId, {}); diff --git a/src/utils/qu.js b/src/utils/qu.js index e566f9ac..08c49c62 100644 --- a/src/utils/qu.js +++ b/src/utils/qu.js @@ -430,6 +430,10 @@ function init(context, selector, window) { const element = selector ? context.querySelector(selector) : context; + if (!element) { + return null; + } + const legacyContextFuncs = Object.entries(legacyFuncs) // dynamically attach methods with context .reduce((acc, [key, func]) => ({ ...acc, diff --git a/src/utils/update.js b/src/utils/update.js new file mode 100644 index 00000000..2f131286 --- /dev/null +++ b/src/utils/update.js @@ -0,0 +1,13 @@ +'use strict'; + +const knex = require('../knex'); + +async function init() { + const result = await knex.raw('SELECT * FROM json_to_recordset(:ids) AS x(id int)', { + ids: JSON.stringify([{ id: 1, foo: 'bar' }, { id: 2 }, { id: 3 }]), + }); + + console.log(result); +} + +init();