DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Reduced media concurrency to assess effect on memory. Moved qu context removal to deep scrape runner. Updated movie graphql queries.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2021-12-05 02:54:55 +01:00
parent 9f37ec4cff
commit e88554666c
5 changed files with 7 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/media.js
View File

@@ -377,6 +377,7 @@ async function writeLazy(image, lazypath) {
async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath, options) {
logger.silly(`Storing permanent media files for ${media.id} from ${media.src} at ${filepath}`);
logger.debug(`Memory usage at image storage: ${process.memoryUsage.rss() / 1000000} MB (${media.src})`);
try {
const thumbdir = config.s3.enabled ? path.join(media.role, 'thumbs') : path.join(media.role, 'thumbs', hashDir, hashSubDir);
@@ -746,7 +747,8 @@ async function storeMedias(baseMedias, options) {
const fetchedMedias = await Promise.map(
baseMedias,
async (baseMedia) => fetchMedia(baseMedia, { existingSourceMediaByUrl, existingExtractMediaByUrl }),
{ concurrency: 100 }, // don't overload disk (or network, although this has its own throttling)
// { concurrency: 100 }, // don't overload disk (or network, although this has its own throttling)
{ concurrency: 10 }, // don't overload disk (or network, although this has its own throttling)
);
const { uniqueHashMedias, existingHashMedias } = await findHashDuplicates(fetchedMedias);