Bumped to Node 24 and upgraded dependencies.

.nvmrc

@@ -1 +1 @@
-20.19.6
+24.19.0

config/default.js

@@ -44,7 +44,8 @@ module.exports = {
             gold: '★',
             over18: '♥',
         },
-        dateFormat: 'YYYYMMDD',
+        // date-fns v2+ token syntax (was 'YYYYMMDD' under date-fns v1)
+        dateFormat: 'yyyyMMdd',
         truncate: {
             limit: 250,
             truncator: '...',

package.json

@@ -27,40 +27,41 @@
   ],
   "author": "Niels Simenon",
   "license": "ISC",
+  "engines": {
+    "node": ">=24.19.0"
+  },
   "dependencies": {
     "array.prototype.flatten": "^1.2.1",
-    "bhttp": "^1.2.4",
+    "bhttp": "^1.2.8",
-    "blake2": "^4.1.1",
+    "blake2": "^5.0.1",
-    "bluebird": "^3.5.1",
+    "bluebird": "^3.7.2",
     "bottleneck": "^2.19.5",
-    "cheerio": "^1.0.0-rc.2",
+    "cheerio": "^1.2.0",
-    "config": "^1.30.0",
+    "config": "^4.4.2",
-    "date-fns": "^1.29.0",
+    "date-fns": "^4.4.0",
     "dist-exiftool": "^10.53.0",
-    "fluent-ffmpeg": "^2.1.2",
+    "fluent-ffmpeg": "^2.1.3",
-    "fs-extra": "^5.0.0",
+    "fs-extra": "^11.4.0",
-    "js-yaml": "^3.12.0",
+    "js-yaml": "^5.2.3",
-    "jsdom": "^15.2.0",
     "mime": "^3.0.0",
-    "mime-types": "^2.1.18",
+    "mime-types": "^3.0.2",
-    "node-cron": "^1.2.1",
+    "node-cron": "^4.6.0",
     "node-exiftool": "^2.3.0",
-    "node-fetch": "^2.1.2",
     "object.omit": "^3.0.0",
     "object.pick": "^1.3.0",
-    "sharp-phash": "^2.1.0",
+    "sharp-phash": "^2.2.0",
-    "template-format": "^1.2.4",
+    "template-format": "^1.2.5",
-    "unprint": "^0.8.2",
+    "unprint": "^0.19.20",
     "url-pattern": "^1.0.3",
-    "winston": "^3.3.3",
+    "winston": "^3.19.0",
-    "winston-daily-rotate-file": "^4.7.1",
+    "winston-daily-rotate-file": "^5.0.0",
-    "yargs": "^11.0.0",
+    "yargs": "^17.7.2",
-    "youtube-dl-exec": "^2.2.3"
+    "youtube-dl-exec": "^3.1.12"
   },
   "devDependencies": {
-    "@babel/eslint-parser": "^7.19.1",
+    "@babel/eslint-parser": "^7.29.7",
-    "eslint": "^8.34.0",
+    "eslint": "^8.57.1",
     "eslint-config-airbnb-base": "^15.0.0",
-    "eslint-plugin-import": "^2.27.5"
+    "eslint-plugin-import": "^2.32.0"
   }
 }

src/app.js

@@ -140,8 +140,10 @@ function fetchSavePosts(userPosts, ep) {
 }
 
 async function initApp() {
+    let ep;
+
     try {
-        const ep = new exiftool.ExiftoolProcess(exiftoolBin);
+        ep = new exiftool.ExiftoolProcess(exiftoolBin);
         await ep.open();
 
         if (args.fetch || args.fileDirect) {
@@ -154,8 +156,6 @@ async function initApp() {
             await fetchSavePosts(userPosts, ep);
         }
 
-        await ep.close();
-
         if (args.watch) {
             logger.info(`Watch-mode enabled, checking again for new posts according to crontab '${config.fetch.watch.schedule}'.`);
         }
@@ -165,6 +165,13 @@ async function initApp() {
         } else {
             logger.error(error.message);
         }
+    } finally {
+        // the exiftool child process keeps the event loop alive - if it's never closed
+        // (e.g. because an error above was thrown before we got to it), the process hangs
+        // forever after logging the error instead of actually exiting
+        if (ep) {
+            await ep.close().catch((error) => logger.error(`Could not close exiftool process: ${error.message}`));
+        }
     }
 }
 

src/archives/ip.js

@@ -1,6 +1,5 @@
 'use strict';
 
-const fetch = require('node-fetch');
 const $ = require('cheerio');
 
 function findOnIp(username, page = 1, acc = []) {

src/cli.js

@@ -124,6 +124,11 @@ function getArgs() {
             type: 'boolean',
             default: config.fetch.archives.search,
         })
+        .option('debug', {
+            describe: 'Log full stack traces instead of just error messages',
+            type: 'boolean',
+            default: false,
+        })
         .argv;
 
     return {

src/fetch/content.js

@@ -46,7 +46,7 @@ function selfPostToText(item, post) {
         comments: curateComments(item.comments),
     };
 
-    return yaml.safeDump(curatedPost);
+    return yaml.dump(curatedPost);
 }
 
 async function getBuffers(item, context) {
@@ -117,38 +117,51 @@ async function fetchSaveUserContent(user, ep, args) {
 
         const items = await Promise.map(post.content.items, async (originalItem, index) => {
             const item = { ...originalItem, index };
-            const buffers = await getBuffers(item, { post, host: post.host, headers: post.content.headers || item.headers });
 
-            // no buffers, ignore item
+            try {
-            if (!buffers || buffers.length === 0) {
+                const buffers = await getBuffers(item, { post, host: post.host, headers: post.content.headers || item.headers });
+
+                // no buffers, ignore item
+                if (!buffers || buffers.length === 0) {
+                    return null;
+                }
+
+                item.hash = buffers[0].hash;
+                item.phash = buffers[0].phash;
+
+                // prevent duplicates
+                if (config.fetch.avoidDuplicates && hashes.has(buffers[0].hash) && !item.album) {
+                    logger.verbose(`Ignoring duplicate file '${post.url}' (${post.permalink})`);
+
+                    return {
+                        ...item,
+                        ignored: true,
+                    };
+                }
+
+                const filepath = getFilepath(item, post.content, post.host, post, user);
+                const sourcePaths = await save(filepath, buffers.map(({ buffer }) => buffer), item, post);
+
+                hashes.add(buffers[0].hash);
+
+                if (item.mux) {
+                    await mux(filepath, sourcePaths, item);
+                }
+
+                await addMeta(filepath, item, post, user, ep);
+
+                return item;
+            } catch (error) {
+                // don't let one bad item (e.g. a post with a missing/invalid date) abort the
+                // entire run - log it with enough context to find it again and move on
+                logger.error(`Failed to fetch/save item ${index} for post '${post.id}' (${post.permalink}): ${error.message}`);
+
+                if (args.debug) {
+                    logger.error(error.stack);
+                }
+
                 return null;
             }
-
-            item.hash = buffers[0].hash;
-            item.phash = buffers[0].phash;
-
-            // prevent duplicates
-            if (config.fetch.avoidDuplicates && hashes.has(buffers[0].hash) && !item.album) {
-                logger.verbose(`Ignoring duplicate file '${post.url}' (${post.permalink})`);
-
-                return {
-                    ...item,
-                    ignored: true,
-                };
-            }
-
-            const filepath = getFilepath(item, post.content, post.host, post, user);
-            const sourcePaths = await save(filepath, buffers.map(({ buffer }) => buffer), item, post);
-
-            hashes.add(buffers[0].hash);
-
-            if (item.mux) {
-                await mux(filepath, sourcePaths, item);
-            }
-
-            await addMeta(filepath, item, post, user, ep);
-
-            return item;
         });
 
         return {

src/interpolate.js

@@ -9,6 +9,24 @@ const format = require('template-format');
 
 const args = require('./cli')();
 
+// date-fns throws a bare, contextless "Invalid time value" RangeError for any invalid date
+// passed to format() - wrap it so a bad timestamp somewhere upstream (a post, item, user or
+// album with a malformed created/createDate field) points back at its source instead.
+// No date at all (e.g. profileDetails.js's item-less-of-a-datetime avatar interpolation) is a
+// legitimate case, not a bug - that just interpolates to an empty string, same as any other
+// unset field like title/description elsewhere in this function.
+function safeFormat(date, dateFormat, label) {
+    if (date === null || date === undefined) {
+        return null;
+    }
+
+    if (!(date instanceof Date) || Number.isNaN(date.getTime())) {
+        throw new Error(`Cannot format ${label} date for filename - invalid datetime (${date})`);
+    }
+
+    return dateFns.format(date, dateFormat);
+}
+
 function interpolate(pattern, item = null, content = null, host = null, post = null, user = null, strip = true, dateFormat = config.library.dateFormat) {
     const data = {
         tags: {},
@@ -20,7 +38,7 @@ function interpolate(pattern, item = null, content = null, host = null, post = n
                 id: item.id,
                 title: item.title && item.title.slice(0, config.library.titleLength),
                 description: item.description,
-                date: dateFns.format(item.datetime, dateFormat),
+                date: safeFormat(item.datetime, dateFormat, 'item'),
                 index: item.index + config.library.indexOffset,
             },
             ext: item.type
@@ -49,7 +67,7 @@ function interpolate(pattern, item = null, content = null, host = null, post = n
                 id: content.album.id,
                 title: content.album.title && content.album.title.slice(0, config.library.titleLength),
                 description: content.album.description,
-                date: dateFns.format(content.album.datetime, dateFormat),
+                date: safeFormat(content.album.datetime, dateFormat, 'album'),
             },
         });
     }
@@ -60,7 +78,7 @@ function interpolate(pattern, item = null, content = null, host = null, post = n
                 id: post.id,
                 title: post.title && post.title.slice(0, config.library.titleLength),
                 url: post.url,
-                date: dateFns.format(post.datetime, dateFormat),
+                date: safeFormat(post.datetime, dateFormat, 'post'),
                 index: post.index + config.library.indexOffset,
                 hash: post.hash,
                 score: post.score,
@@ -75,7 +93,7 @@ function interpolate(pattern, item = null, content = null, host = null, post = n
                 name: user.name,
                 username: user.name,
                 id: user.id,
-                created: dateFns.format(user.created, dateFormat),
+                created: safeFormat(user.created, dateFormat, 'user'),
             },
             tags: {
                 ...data.tags,

src/methods/erome.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const config = require('config');
-const fetch = require('node-fetch');
 const cheerio = require('cheerio');
 
 const logger = require('../logger')(__filename);

src/methods/eroshareAlbum.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const config = require('config');
-const fetch = require('node-fetch');
 
 async function eroshareAlbum(host) {
     const res = await fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/${host.id}`);

src/methods/eroshareItem.js

@@ -1,6 +1,5 @@
 'use strict';
 
-const fetch = require('node-fetch');
 const cheerio = require('cheerio');
 
 async function eroshareItem(host, post) {

src/methods/imgurAlbum.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const config = require('config');
-const fetch = require('node-fetch');
 
 const logger = require('../logger')(__filename);
 const { fetchPredata } = require('./imgurImage');

src/methods/imgurImage.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const config = require('config');
-const fetch = require('node-fetch');
 
 async function fetchPredata() {
     const data = {

src/methods/redditAlbum.js

@@ -1,18 +1,32 @@
 'use strict';
 
-const mime = require('mime');
+// reddit gallery images: `s.u` (still) or `s.gif`/`s.mp4` (animated) - URLs come back
-const bhttp = require('bhttp');
+// HTML-entity-encoded (e.g. '&' instead of '&')
-const { JSDOM } = require('jsdom');
+function resolveMediaUrl(media) {
+    const url = media?.s?.u || media?.s?.gif || media?.s?.mp4;
 
-async function redditAlbum(host, post) {
+    return url ? url.replace(/&/g, '&') : null;
-    const res = await bhttp.get(host.url);
+}
 
-    if (res.statusCode !== 200) {
+async function redditAlbum(host, post, { reddit }) {
-        throw new Error(res.body.toString());
+    // fetched through the authenticated/throttled client, not a raw fetch()/bhttp request -
-    }
+    // reddit's anonymous-traffic bot challenge blocks plain unauthenticated .json requests
+    const data = await reddit.getSubmission(post.id);
 
-    const { document } = new JSDOM(res.body.toString(), { runScripts: 'dangerously' }).window;
+    const items = (data.gallery_data?.items || [])
-    const items = Array.from(document.querySelectorAll('li a'), el => el.href);
+        .filter((item) => !item.is_deleted)
+        .map((item) => {
+            const media = data.media_metadata?.[item.media_id];
+            const url = resolveMediaUrl(media);
+
+            return url && {
+                id: item.media_id,
+                url,
+                datetime: post.datetime,
+                type: media.m || 'image/jpeg',
+            };
+        })
+        .filter(Boolean);
 
     return {
         album: {
@@ -20,12 +34,7 @@ async function redditAlbum(host, post) {
             url: host.url,
             title: post.title,
         },
-        items: items.map((url) => ({
+        items,
-            id: new URL(url).pathname.match(/\/(.*).\w+$/)?.[1],
-            url,
-            datetime: post.datetime,
-            type: mime.getType(url) || 'image/jpeg',
-        })),
     };
 }
 

src/methods/redditVideo.js

@@ -1,23 +1,48 @@
 'use strict';
 
-const fetch = require('node-fetch');
+// the DASH manifest lists the actual audio track filename - reddit has changed this naming
+// scheme before (DASH_audio.mp4 -> /audio -> CMAF_AUDIO_128.mp4), so read it from the
+// manifest rather than guessing a hardcoded suffix off the video URL again
+function findAudioUrl(manifest, videoId) {
+    const audioSection = manifest.match(/<AdaptationSet[^>]*contentType="audio"[^>]*>([\s\S]*?)<\/AdaptationSet>/);
 
-async function redditVideo(host, post) {
+    if (!audioSection) {
-    const res = await fetch(`${post.permalink}.json`);
+        return null;
-    const [{ data }] = await res.json();
+    }
 
-    const videoUrl = data.children[0].data.media.reddit_video.fallback_url;
+    const baseUrls = [...audioSection[1].matchAll(/<BaseURL>([^<]+)<\/BaseURL>/g)].map((match) => match[1]);
-    const audioUrl = `${videoUrl.split('/').slice(0, -1).join('/')}/audio`;
+    // representations are listed lowest to highest bandwidth - take the best one
+    const audioFile = baseUrls[baseUrls.length - 1];
 
-    const audioRes = await fetch(audioUrl, {
+    return audioFile ? `https://v.redd.it/${videoId}/${audioFile}` : null;
-        method: 'HEAD',
+}
-    });
+
+async function findAudio(video, videoId) {
+    if (!video.has_audio || !video.dash_url) {
+        return null;
+    }
+
+    const res = await fetch(video.dash_url);
+
+    if (!res.ok) {
+        return null;
+    }
+
+    return findAudioUrl(await res.text(), videoId);
+}
+
+async function redditVideo(host, post, { reddit }) {
+    // fetched through the authenticated/throttled client, not a raw fetch() - reddit's
+    // anonymous-traffic bot challenge blocks plain unauthenticated .json requests
+    const data = await reddit.getSubmission(post.id);
+    const video = data.media.reddit_video;
+    const videoId = post.host.id || post.id;
 
     const item = {
         album: null,
         items: [{
-            id: post.host.id || post.id,
+            id: videoId,
-            url: videoUrl,
+            url: video.fallback_url,
             title: post.title,
             datetime: post.datetime,
             type: 'video/mp4',
@@ -25,7 +50,9 @@ async function redditVideo(host, post) {
         }],
     };
 
-    if (audioRes.status === 200) {
+    const audioUrl = await findAudio(video, videoId);
+
+    if (audioUrl) {
         item.items[0].mux = [audioUrl];
     }
 

src/methods/redgifs.js

@@ -1,6 +1,5 @@
 'use strict';
 
-const fetch = require('node-fetch');
 const mime = require('mime');
 // const unprint = require('unprint');
 
@@ -28,6 +27,11 @@ async function fetchPredata() {
 }
 
 function scrapeGallery(data, { predata }) {
+    if (!data.gifs.length) {
+        // Math.min() of an empty array is Infinity, which becomes an invalid date below
+        return null;
+    }
+
     const oldestDate = Math.min(...data.gifs.map((gif) => gif.createDate));
 
     const curated = {

src/methods/tube.js

@@ -25,7 +25,7 @@ async function tube(host, post) {
                     title: data.fulltitle || data.title,
                     description: data.description,
                     type: `video/${data.ext}`,
-                    datetime: dateFns.parse(data.upload_date, 'YYYYMMDD'),
+                    datetime: dateFns.parse(data.upload_date, 'yyyyMMdd', new Date()),
                     original: data,
                 },
             ],

src/methods/vidbleAlbum.js

@@ -1,7 +1,6 @@
 'use strict';
 
 const config = require('config');
-const fetch = require('node-fetch');
 const UrlPattern = require('url-pattern');
 const cheerio = require('cheerio');
 const mime = require('mime-types');

src/methods/vidbleImage.js

@@ -1,6 +1,5 @@
 'use strict';
 
-const fetch = require('node-fetch');
 const $ = require('cheerio');
 const mime = require('mime-types');
 

src/methods/vidbleVideo.js

@@ -1,6 +1,5 @@
 'use strict';
 
-const fetch = require('node-fetch');
 const cheerio = require('cheerio');
 
 async function vidbleVideo(host, post) {

src/reddit/client.js

@@ -1,6 +1,5 @@
 'use strict';
 
-const fetch = require('node-fetch');
 const Bottleneck = require('bottleneck');
 const config = require('config');
 

src/save/mux.js

@@ -18,7 +18,7 @@ async function mux(target, sources) {
 
                 resolve(stdout);
             })
-            .on('error', () => reject)
+            .on('error', (error) => reject(error))
             .save(target);
     }).then(() => Promise.all(sources.map((source) => fs.remove(source))).then(() => {
         logger.verbose(`Cleaned up temporary files for '${target}'`);

src/save/writeToIndex.js

@@ -61,7 +61,7 @@ async function writeToIndex(posts, profilePaths, user, args) {
     }
 
     try {
-        const yamlIndex = yaml.safeDump(data, { skipInvalid: true });
+        const yamlIndex = yaml.dump(data, { skipInvalid: true });
         const saved = await save(filepath, Buffer.from(yamlIndex, 'utf8'));
 
         logger.info(`Saved index with ${posts.length} new posts for ${user.name}`);

src/sources/getIndex.js

@@ -13,7 +13,7 @@ async function getIndex(user) {
     try {
         const indexFile = await fs.readFile(indexFilePath, 'utf8');
 
-        return yaml.safeLoad(indexFile, { json: true }); // allow duplicate keys
+        return yaml.load(indexFile, { json: true }); // allow duplicate keys
     } catch (error) {
         logger.info(`No index file found for '${user.name}' at '${indexFilePath}': ${error.message}`);