DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Reusing batch ID for movies to preserve new-flag.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2022-01-20 00:54:10 +01:00
parent 372db86927
commit 20da2d1cf6
6 changed files with 16 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/app.js
View File

@@ -186,8 +186,8 @@ async function init() {
}
if (argv.save) {
const storedMovies = await storeMovies(deepMovies);
const storedMovieScenes = await storeScenes(deepMovieScenes);
const storedMovies = await storeMovies(deepMovies, storedScenes[0]?.batchId);
const storedMovieScenes = await storeScenes(deepMovieScenes, storedScenes[0]?.batchId);
await associateMovieScenes(storedMovies, [...storedScenes, ...storedMovieScenes]);
}

5
src/scrapers/mindgeek.js
View File

@@ -238,8 +238,7 @@ async function fetchLatest(site, page = 1, options) {
const { searchParams } = new URL(url);
const siteId = searchParams.get('site');
// const { session, instanceToken } = options.beforeNetwork?.headers?.Instance ? options.beforeNetwork : await getSession(site, options.parameters);
const { session, instanceToken } = await getSession(site, options.parameters);
const { session, instanceToken } = options.beforeNetwork?.headers?.Instance ? options.beforeNetwork : await getSession(site, options.parameters);
const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
const limit = 24;
@@ -365,7 +364,7 @@ async function fetchProfile({ name: actorName, slug: actorSlug }, { entity, para
}
module.exports = {
// beforeNetwork: getSession,
beforeNetwork: getSession,
beforeFetchScenes: getSession,
scrapeLatestX,
fetchLatest,

15
src/store-releases.js
View File

@@ -136,7 +136,7 @@ async function attachStudios(releases) {
return releasesWithStudio;
}
function attachReleaseIds(releases, storedReleases) {
function attachReleaseIds(releases, storedReleases, batchId) {
const storedReleaseIdsByEntityIdAndEntryId = storedReleases.reduce((acc, release) => {
if (!acc[release.entity_id]) acc[release.entity_id] = {};
acc[release.entity_id][release.entry_id] = release.id;
@@ -157,6 +157,7 @@ function attachReleaseIds(releases, storedReleases) {
return {
...release,
id,
batchId,
};
}
@@ -314,12 +315,12 @@ async function storeChapters(releases) {
await associateReleaseMedia(chaptersWithId, 'chapter');
}
async function storeScenes(releases) {
async function storeScenes(releases, useBatchId) {
if (!releases || releases.length === 0) {
return [];
}
const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id');
const releasesWithChannels = await attachChannelEntities(releases);
const releasesWithBaseActors = releasesWithChannels.map((release) => ({ ...release, actors: toBaseActors(release.actors) }));
@@ -333,8 +334,8 @@ async function storeScenes(releases) {
const storedReleaseEntries = Array.isArray(storedReleases) ? storedReleases : [];
const uniqueReleasesWithId = attachReleaseIds(uniqueReleases, storedReleaseEntries);
const duplicateReleasesWithId = attachReleaseIds(duplicateReleases, duplicateReleaseEntries);
const uniqueReleasesWithId = attachReleaseIds(uniqueReleases, storedReleaseEntries, batchId);
const duplicateReleasesWithId = attachReleaseIds(duplicateReleases, duplicateReleaseEntries, batchId);
const releasesWithId = uniqueReleasesWithId.concat(duplicateReleasesWithId);
const updated = await knex.raw(`
@@ -444,13 +445,13 @@ async function updateMovieSearch(movieIds) {
}
}
async function storeMovies(movies) {
async function storeMovies(movies, useBatchId) {
if (!movies || movies.length === 0) {
return [];
}
const { uniqueReleases } = await filterDuplicateReleases(movies);
const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id');
const curatedMovieEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId, null, 'movie')));