DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Using line-separated JSON to save memory in transfer tool.

This commit is contained in:
DebaucheryLibrarian 2023-06-03 22:41:15 +02:00
parent f3abc21482
commit f356135722
1 changed files with 48 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
src/tools/transfer.js
View File

@ -260,38 +260,54 @@ async function save() {
offset, offset,
}, 'owner'); }, 'owner');
const curatedScenes = releases.map((release) => ({
...release,
type: 'release',
actors: release.actors.filter(Boolean).map(({ actor }) => actor),
directors: release.directors.filter(Boolean).map(({ director }) => director),
studio: release.studio?.slug,
tags: release.tags.map(({ tag }) => tag?.slug).filter(Boolean),
movies: release.movies?.map(({ movie }) => movie) || [],
chapters: release.chapters.filter(Boolean).map((chapter) => ({
...chapter,
tags: chapter.tags.map(({ tag }) => tag?.slug).filter(Boolean),
})),
poster: release.poster?.media,
trailer: release.trailer?.media,
teaser: release.teaser?.media,
photos: release.photos.filter(Boolean).map(({ media }) => media),
covers: release.covers.filter(Boolean).map(({ media }) => media),
}));
const curatedMovies = movies.map((movie) => ({
...movie,
type: 'movie',
poster: movie.poster?.media,
covers: movie.covers.filter(Boolean).map(({ media }) => media),
}));
const filename = `export-${offset}-${offset + limit}-${moment().format('YYYY-MM-DD_hh_mm_ss')}.json`; const filename = `export-${offset}-${offset + limit}-${moment().format('YYYY-MM-DD_hh_mm_ss')}.json`;
const serializedData = JSON.stringify([...curatedScenes, ...curatedMovies], null, 4);
await fs.promises.writeFile(filename, serializedData); let savedScenes = 0;
let savedMovies = 0;
console.log(`Saved ${releases.length} releases to ${filename}`); await releases.reduce(async (chain, release) => {
await chain;
const entry = JSON.stringify({
...release,
type: 'release',
actors: release.actors.filter(Boolean).map(({ actor }) => actor),
directors: release.directors.filter(Boolean).map(({ director }) => director),
studio: release.studio?.slug,
tags: release.tags.map(({ tag }) => tag?.slug).filter(Boolean),
movies: release.movies?.map(({ movie }) => movie) || [],
chapters: release.chapters.filter(Boolean).map((chapter) => ({
...chapter,
tags: chapter.tags.map(({ tag }) => tag?.slug).filter(Boolean),
})),
poster: release.poster?.media,
trailer: release.trailer?.media,
teaser: release.teaser?.media,
photos: release.photos.filter(Boolean).map(({ media }) => media),
covers: release.covers.filter(Boolean).map(({ media }) => media),
});
await fs.promises.appendFile(filename, `${entry}\n`);
savedScenes += 1;
}, Promise.resolve());
await movies.reduce(async (chain, movie) => {
await chain;
const entry = JSON.stringify({
...movie,
type: 'movie',
poster: movie.poster?.media,
covers: movie.covers.filter(Boolean).map(({ media }) => media),
});
await fs.promises.appendFile(filename, `${entry}\n`);
savedMovies += 1;
}, Promise.resolve());
console.log(`Saved ${savedScenes} scenes and ${savedMovies} movies to ${filename}`);
process.exit(); process.exit();
} }
@ -555,7 +571,9 @@ async function addRelease(release, context) {
async function load() { async function load() {
const file = await fs.promises.readFile(args.file, 'utf8'); const file = await fs.promises.readFile(args.file, 'utf8');
const releases = JSON.parse(file) const releases = file.split('\n')
.filter(Boolean)
.map((data) => JSON.parse(data))
.filter((release) => (args.entity ? release.entity.slug === args.entity : true)) .filter((release) => (args.entity ? release.entity.slug === args.entity : true))
.slice(0, args.limit || Infinity); .slice(0, args.limit || Infinity);