diff --git a/.gitignore b/.gitignore index d87f3907..9eb45e8d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ config/* assets/js/config/ !assets/js/config/default.js *.heapprofile +*.heapsnapshot diff --git a/migrations/20190325001339_releases.js b/migrations/20190325001339_releases.js index 094ab822..336c1118 100644 --- a/migrations/20190325001339_releases.js +++ b/migrations/20190325001339_releases.js @@ -1,6 +1,6 @@ const config = require('config'); -exports.up = knex => Promise.resolve() +exports.up = (knex) => Promise.resolve() .then(() => knex.schema.createTable('countries', (table) => { table.text('alpha2', 2) .unique() diff --git a/migrations/20211030162955_entity_scenes.js b/migrations/20211030162955_entity_scenes.js index 24a82252..af72175e 100644 --- a/migrations/20211030162955_entity_scenes.js +++ b/migrations/20211030162955_entity_scenes.js @@ -1,4 +1,4 @@ -exports.up = async knex => knex.raw(` +exports.up = async (knex) => knex.raw(` CREATE FUNCTION entities_scenes(entity entities) RETURNS SETOF releases AS $$ WITH RECURSIVE children AS ( SELECT entities.id @@ -19,6 +19,6 @@ exports.up = async knex => knex.raw(` COMMENT ON FUNCTION entities_scenes IS E'@sortable'; `); -exports.down = async knex => knex.raw(` +exports.down = async (knex) => knex.raw(` DROP FUNCTION IF EXISTS entities_scenes; `); diff --git a/src/app.js b/src/app.js index 85d1d8d8..c7d418d8 100644 --- a/src/app.js +++ b/src/app.js @@ -186,8 +186,8 @@ async function init() { } if (argv.save) { - const storedMovies = await storeMovies(deepMovies); - const storedMovieScenes = await storeScenes(deepMovieScenes); + const storedMovies = await storeMovies(deepMovies, storedScenes[0]?.batchId); + const storedMovieScenes = await storeScenes(deepMovieScenes, storedScenes[0]?.batchId); await associateMovieScenes(storedMovies, [...storedScenes, ...storedMovieScenes]); } diff --git a/src/scrapers/mindgeek.js b/src/scrapers/mindgeek.js index f4db2a21..9c4917b8 100644 --- a/src/scrapers/mindgeek.js +++ b/src/scrapers/mindgeek.js @@ -238,8 +238,7 @@ async function fetchLatest(site, page = 1, options) { const { searchParams } = new URL(url); const siteId = searchParams.get('site'); - // const { session, instanceToken } = options.beforeNetwork?.headers?.Instance ? options.beforeNetwork : await getSession(site, options.parameters); - const { session, instanceToken } = await getSession(site, options.parameters); + const { session, instanceToken } = options.beforeNetwork?.headers?.Instance ? options.beforeNetwork : await getSession(site, options.parameters); const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD'); const limit = 24; @@ -365,7 +364,7 @@ async function fetchProfile({ name: actorName, slug: actorSlug }, { entity, para } module.exports = { - // beforeNetwork: getSession, + beforeNetwork: getSession, beforeFetchScenes: getSession, scrapeLatestX, fetchLatest, diff --git a/src/store-releases.js b/src/store-releases.js index 804ea746..b94a5bb0 100644 --- a/src/store-releases.js +++ b/src/store-releases.js @@ -136,7 +136,7 @@ async function attachStudios(releases) { return releasesWithStudio; } -function attachReleaseIds(releases, storedReleases) { +function attachReleaseIds(releases, storedReleases, batchId) { const storedReleaseIdsByEntityIdAndEntryId = storedReleases.reduce((acc, release) => { if (!acc[release.entity_id]) acc[release.entity_id] = {}; acc[release.entity_id][release.entry_id] = release.id; @@ -157,6 +157,7 @@ function attachReleaseIds(releases, storedReleases) { return { ...release, id, + batchId, }; } @@ -314,12 +315,12 @@ async function storeChapters(releases) { await associateReleaseMedia(chaptersWithId, 'chapter'); } -async function storeScenes(releases) { +async function storeScenes(releases, useBatchId) { if (!releases || releases.length === 0) { return []; } - const [batchId] = await knex('batches').insert({ comment: null }).returning('id'); + const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id'); const releasesWithChannels = await attachChannelEntities(releases); const releasesWithBaseActors = releasesWithChannels.map((release) => ({ ...release, actors: toBaseActors(release.actors) })); @@ -333,8 +334,8 @@ async function storeScenes(releases) { const storedReleaseEntries = Array.isArray(storedReleases) ? storedReleases : []; - const uniqueReleasesWithId = attachReleaseIds(uniqueReleases, storedReleaseEntries); - const duplicateReleasesWithId = attachReleaseIds(duplicateReleases, duplicateReleaseEntries); + const uniqueReleasesWithId = attachReleaseIds(uniqueReleases, storedReleaseEntries, batchId); + const duplicateReleasesWithId = attachReleaseIds(duplicateReleases, duplicateReleaseEntries, batchId); const releasesWithId = uniqueReleasesWithId.concat(duplicateReleasesWithId); const updated = await knex.raw(` @@ -444,13 +445,13 @@ async function updateMovieSearch(movieIds) { } } -async function storeMovies(movies) { +async function storeMovies(movies, useBatchId) { if (!movies || movies.length === 0) { return []; } const { uniqueReleases } = await filterDuplicateReleases(movies); - const [batchId] = await knex('batches').insert({ comment: null }).returning('id'); + const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id'); const curatedMovieEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId, null, 'movie')));