DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Replaced bhttp with patched fork. Improved Jesse Loads Monster Facials scraper reliability (WIP). Added various tag photos.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2020-10-30 17:37:10 +01:00
parent 4af7597441
commit 39f8c037a5
43 changed files with 128 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/media.js
View File

@@ -279,6 +279,8 @@ async function extractSource(baseSource, { existingExtractMediaByUrl }) {
}
async function storeImageFile(media, hashDir, hashSubDir, filename, filedir, filepath) {
logger.silly(`Storing permanent media files for ${media.id} from ${media.src} at ${filepath}`);
try {
const thumbdir = path.join(media.role, 'thumbs', hashDir, hashSubDir);
const thumbpath = path.join(thumbdir, filename);
@@ -620,6 +622,7 @@ async function storeMedias(baseMedias) {
const fetchedMedias = await Promise.map(
baseMedias,
async baseMedia => fetchMedia(baseMedia, { existingSourceMediaByUrl, existingExtractMediaByUrl }),
{ concurrency: 100 }, // don't overload disk (or network, although this has its own throttling)
);
const [uniqueHashMedias, existingHashMedias] = await findHashDuplicates(fetchedMedias);
@@ -627,6 +630,7 @@ async function storeMedias(baseMedias) {
const savedMedias = await Promise.map(
uniqueHashMedias,
async baseMedia => storeFile(baseMedia),
{ concurrency: 100 }, // don't overload disk
);
if (argv.force) {
@@ -634,6 +638,7 @@ async function storeMedias(baseMedias) {
await Promise.map(
existingHashMedias,
async baseMedia => storeFile(baseMedia),
{ concurrency: 100 }, // don't overload disk
);
}
@@ -784,7 +789,7 @@ async function flushOrphanedMedia() {
await fsPromises.rmdir(path.join(config.media.path, 'temp'), { recursive: true });
logger.info('Removed temporary media directory');
logger.info('Cleared temporary media directory');
}
module.exports = {