DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Using line-separated JSON to save memory in transfer tool.

This commit is contained in:
DebaucheryLibrarian 2023-06-03 22:41:15 +02:00
parent f3abc21482
commit f356135722
1 changed files with 48 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/tools/transfer.js
View File

@ -260,7 +260,15 @@ async function save() {
offset,
}, 'owner');
const curatedScenes = releases.map((release) => ({
const filename = `export-${offset}-${offset + limit}-${moment().format('YYYY-MM-DD_hh_mm_ss')}.json`;
let savedScenes = 0;
let savedMovies = 0;
await releases.reduce(async (chain, release) => {
await chain;
const entry = JSON.stringify({
...release,
type: 'release',
actors: release.actors.filter(Boolean).map(({ actor }) => actor),
@ -277,21 +285,29 @@ async function save() {
teaser: release.teaser?.media,
photos: release.photos.filter(Boolean).map(({ media }) => media),
covers: release.covers.filter(Boolean).map(({ media }) => media),
}));
});
const curatedMovies = movies.map((movie) => ({
await fs.promises.appendFile(filename, `${entry}\n`);
savedScenes += 1;
}, Promise.resolve());
await movies.reduce(async (chain, movie) => {
await chain;
const entry = JSON.stringify({
...movie,
type: 'movie',
poster: movie.poster?.media,
covers: movie.covers.filter(Boolean).map(({ media }) => media),
}));
});
const filename = `export-${offset}-${offset + limit}-${moment().format('YYYY-MM-DD_hh_mm_ss')}.json`;
const serializedData = JSON.stringify([...curatedScenes, ...curatedMovies], null, 4);
await fs.promises.appendFile(filename, `${entry}\n`);
await fs.promises.writeFile(filename, serializedData);
savedMovies += 1;
}, Promise.resolve());
console.log(`Saved ${releases.length} releases to ${filename}`);
console.log(`Saved ${savedScenes} scenes and ${savedMovies} movies to ${filename}`);
process.exit();
}
@ -555,7 +571,9 @@ async function addRelease(release, context) {
async function load() {
const file = await fs.promises.readFile(args.file, 'utf8');
const releases = JSON.parse(file)
const releases = file.split('\n')
.filter(Boolean)
.map((data) => JSON.parse(data))
.filter((release) => (args.entity ? release.entity.slug === args.entity : true))
.slice(0, args.limit || Infinity);