DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Reusing batch ID for movies to preserve new-flag.

This commit is contained in:
DebaucheryLibrarian 2022-01-20 00:54:10 +01:00
parent 372db86927
commit 20da2d1cf6
6 changed files with 16 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -10,3 +10,4 @@ config/*
assets/js/config/
!assets/js/config/default.js
*.heapprofile
*.heapsnapshot

2
migrations/20190325001339_releases.js
View File

@ -1,6 +1,6 @@
const config = require('config');
exports.up = knex => Promise.resolve()
exports.up = (knex) => Promise.resolve()
.then(() => knex.schema.createTable('countries', (table) => {
table.text('alpha2', 2)
.unique()

4
migrations/20211030162955_entity_scenes.js
View File

@ -1,4 +1,4 @@
exports.up = async knex => knex.raw(`
exports.up = async (knex) => knex.raw(`
CREATE FUNCTION entities_scenes(entity entities) RETURNS SETOF releases AS $$
WITH RECURSIVE children AS (
SELECT entities.id
@ -19,6 +19,6 @@ exports.up = async knex => knex.raw(`
COMMENT ON FUNCTION entities_scenes IS E'@sortable';
`);
exports.down = async knex => knex.raw(`
exports.down = async (knex) => knex.raw(`
DROP FUNCTION IF EXISTS entities_scenes;
`);

4
src/app.js
View File

@ -186,8 +186,8 @@ async function init() {
}
if (argv.save) {
const storedMovies = await storeMovies(deepMovies);
const storedMovieScenes = await storeScenes(deepMovieScenes);
const storedMovies = await storeMovies(deepMovies, storedScenes[0]?.batchId);
const storedMovieScenes = await storeScenes(deepMovieScenes, storedScenes[0]?.batchId);
await associateMovieScenes(storedMovies, [...storedScenes, ...storedMovieScenes]);
}

5
src/scrapers/mindgeek.js
View File

@ -238,8 +238,7 @@ async function fetchLatest(site, page = 1, options) {
const { searchParams } = new URL(url);
const siteId = searchParams.get('site');
// const { session, instanceToken } = options.beforeNetwork?.headers?.Instance ? options.beforeNetwork : await getSession(site, options.parameters);
const { session, instanceToken } = await getSession(site, options.parameters);
const { session, instanceToken } = options.beforeNetwork?.headers?.Instance ? options.beforeNetwork : await getSession(site, options.parameters);
const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
const limit = 24;
@ -365,7 +364,7 @@ async function fetchProfile({ name: actorName, slug: actorSlug }, { entity, para
}
module.exports = {
// beforeNetwork: getSession,
beforeNetwork: getSession,
beforeFetchScenes: getSession,
scrapeLatestX,
fetchLatest,

15
src/store-releases.js
View File

@ -136,7 +136,7 @@ async function attachStudios(releases) {
return releasesWithStudio;
}
function attachReleaseIds(releases, storedReleases) {
function attachReleaseIds(releases, storedReleases, batchId) {
const storedReleaseIdsByEntityIdAndEntryId = storedReleases.reduce((acc, release) => {
if (!acc[release.entity_id]) acc[release.entity_id] = {};
acc[release.entity_id][release.entry_id] = release.id;
@ -157,6 +157,7 @@ function attachReleaseIds(releases, storedReleases) {
return {
...release,
id,
batchId,
};
}
@ -314,12 +315,12 @@ async function storeChapters(releases) {
await associateReleaseMedia(chaptersWithId, 'chapter');
}
async function storeScenes(releases) {
async function storeScenes(releases, useBatchId) {
if (!releases || releases.length === 0) {
return [];
}
const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id');
const releasesWithChannels = await attachChannelEntities(releases);
const releasesWithBaseActors = releasesWithChannels.map((release) => ({ ...release, actors: toBaseActors(release.actors) }));
@ -333,8 +334,8 @@ async function storeScenes(releases) {
const storedReleaseEntries = Array.isArray(storedReleases) ? storedReleases : [];
const uniqueReleasesWithId = attachReleaseIds(uniqueReleases, storedReleaseEntries);
const duplicateReleasesWithId = attachReleaseIds(duplicateReleases, duplicateReleaseEntries);
const uniqueReleasesWithId = attachReleaseIds(uniqueReleases, storedReleaseEntries, batchId);
const duplicateReleasesWithId = attachReleaseIds(duplicateReleases, duplicateReleaseEntries, batchId);
const releasesWithId = uniqueReleasesWithId.concat(duplicateReleasesWithId);
const updated = await knex.raw(`
@ -444,13 +445,13 @@ async function updateMovieSearch(movieIds) {
}
}
async function storeMovies(movies) {
async function storeMovies(movies, useBatchId) {
if (!movies || movies.length === 0) {
return [];
}
const { uniqueReleases } = await filterDuplicateReleases(movies);
const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
const [batchId] = useBatchId ? [useBatchId] : await knex('batches').insert({ comment: null }).returning('id');
const curatedMovieEntries = await Promise.all(uniqueReleases.map((release) => curateReleaseEntry(release, batchId, null, 'movie')));