DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 38 Pull Requests 1 Releases Wiki Activity

Improved movie scraping.

This commit is contained in:
DebaucheryLibrarian 2021-01-29 02:38:05 +01:00
parent 4d89256a4c
commit b7aaeada45
2 changed files with 25 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/app.js
View File

@ -8,7 +8,7 @@ const initServer = require('./web/server');
const knex = require('./knex'); const knex = require('./knex');
const fetchUpdates = require('./updates'); const fetchUpdates = require('./updates');
const { fetchScenes, fetchMovies } = require('./deep'); const { fetchScenes, fetchMovies } = require('./deep');
const { storeScenes, storeMovies, updateReleasesSearch } = require('./store-releases'); const { storeScenes, storeMovies, updateReleasesSearch, associateMovieScenes } = require('./store-releases');
const { scrapeActors, flushActors, flushProfiles, interpolateProfiles } = require('./actors'); const { scrapeActors, flushActors, flushProfiles, interpolateProfiles } = require('./actors');
const { flushEntities } = require('./entities'); const { flushEntities } = require('./entities');
const { deleteScenes, deleteMovies, flushScenes, flushBatches } = require('./releases'); const { deleteScenes, deleteMovies, flushScenes, flushBatches } = require('./releases');
@ -79,7 +79,7 @@ async function init() {
const sceneMovies = deepScenes ? deepScenes.map(scene => ({ ...scene.movie, entity: scene.entity })).filter(Boolean) : []; const sceneMovies = deepScenes ? deepScenes.map(scene => ({ ...scene.movie, entity: scene.entity })).filter(Boolean) : [];
const deepMovies = argv.sceneMovies || argv.movie ? await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]) : sceneMovies; const deepMovies = argv.sceneMovies || argv.movie ? await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]) : sceneMovies;
const movieScenes = argv.movieScenes ? deepMovies.map(movie => movie.scenes?.map(scene => ({ ...scene, entity: movie.entity }))).flat().filter(Boolean) : []; const movieScenes = argv.movieScenes ? deepMovies.map(movie => movie.scenes?.map(scene => ({ ...scene, movie, entity: movie.entity }))).flat().filter(Boolean) : [];
const deepMovieScenes = argv.deep ? await fetchScenes(movieScenes) : movieScenes; const deepMovieScenes = argv.deep ? await fetchScenes(movieScenes) : movieScenes;
if (argv.report) { if (argv.report) {
@ -88,15 +88,10 @@ async function init() {
} }
if (argv.save) { if (argv.save) {
const storedScenes = deepScenes.length > 0 || deepMovieScenes.length > 0 const storedMovies = await storeMovies(deepMovies);
? await storeScenes(deepScenes) const storedScenes = await storeScenes([...(deepScenes || []), ...(deepMovieScenes || [])]);
: [];
if (deepMovies.length > 0) { await associateMovieScenes(storedMovies, storedScenes);
const storedMovieScenes = await storeScenes(deepMovieScenes);
await storeMovies(deepMovies, [...(storedMovieScenes || []), ...storedScenes]);
}
} }
knex.destroy(); knex.destroy();

46
src/store-releases.js
View File

@ -290,7 +290,7 @@ async function storeClips(releases) {
} }
async function storeScenes(releases) { async function storeScenes(releases) {
if (releases.length === 0) { if (!releases || releases.length === 0) {
return []; return [];
} }
@ -331,42 +331,36 @@ async function storeScenes(releases) {
} }
async function associateMovieScenes(movies, movieScenes) { async function associateMovieScenes(movies, movieScenes) {
const movieScenesByEntityIdAndEntryId = movieScenes.reduce((acc, scene) => ({ const moviesByEntityIdAndEntryId = movies.reduce((acc, movie) => ({
...acc, ...acc,
[scene.entity.id]: { [movie.entity.id]: {
...acc[scene.entity.id], ...acc[movie.entity.id],
[scene.entryId]: scene, [movie.entryId]: movie,
}, },
}), {}); }), {});
console.log('movies', movies, movieScenes); const associations = movieScenes.map((scene) => {
if (!scene.movie) {
const associations = movies.map((movie) => {
if (!movie.scenes || !movie.id) {
return null; return null;
} }
console.log(movie, movie.scenes); const sceneMovie = moviesByEntityIdAndEntryId[scene.entity.id]?.[scene.movie.entryId];
return movie.scenes.map((scene) => { if (sceneMovie?.id) {
const movieScene = movieScenesByEntityIdAndEntryId[movie.entity.id]?.[scene.entryId]; return {
movie_id: sceneMovie.id,
scene_id: scene.id,
};
}
if (movieScene?.id) { return null;
return { }).filter(Boolean);
movie_id: movie.id,
scene_id: movieScene.id,
};
}
return null;
});
}).flat().filter(Boolean);
await bulkInsert('movies_scenes', associations, false); await bulkInsert('movies_scenes', associations, false);
} }
async function storeMovies(movies, movieScenes) { async function storeMovies(movies) {
if (movies.length === 0) { if (!movies || movies.length === 0) {
return []; return [];
} }
@ -378,13 +372,13 @@ async function storeMovies(movies, movieScenes) {
const storedMovies = await bulkInsert('movies', curatedMovieEntries, ['entity_id', 'entry_id'], true); const storedMovies = await bulkInsert('movies', curatedMovieEntries, ['entity_id', 'entry_id'], true);
const moviesWithId = attachReleaseIds(movies, storedMovies); const moviesWithId = attachReleaseIds(movies, storedMovies);
await associateMovieScenes(moviesWithId, movieScenes);
await associateReleaseMedia(moviesWithId, 'movie'); await associateReleaseMedia(moviesWithId, 'movie');
return storedMovies; return moviesWithId;
} }
module.exports = { module.exports = {
associateMovieScenes,
storeScenes, storeScenes,
storeMovies, storeMovies,
updateReleasesSearch, updateReleasesSearch,