DebaucheryLibrarian/ripunzel
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Bumped to Node 24 and upgraded dependencies.

Browse Source
This commit is contained in:
DebaucheryLibrarian
2026-08-09 02:03:54 +02:00
parent 2e4ba27d98
commit 5c7a0397c9
25 changed files with 5340 additions and 8905 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.nvmrc
View File

@@ -1 +1 @@
20.19.6
24.19.0

3
config/default.js
View File

@@ -44,7 +44,8 @@ module.exports = {
gold: '★',
over18: '♥',
},
dateFormat: 'YYYYMMDD',
// date-fns v2+ token syntax (was 'YYYYMMDD' under date-fns v1)
dateFormat: 'yyyyMMdd',
truncate: {
limit: 250,
truncator: '...',

13960
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

47
package.json
View File

@@ -27,40 +27,41 @@
],
"author": "Niels Simenon",
"license": "ISC",
"engines": {
"node": ">=24.19.0"
},
"dependencies": {
"array.prototype.flatten": "^1.2.1",
"bhttp": "^1.2.4",
"blake2": "^4.1.1",
"bluebird": "^3.5.1",
"bhttp": "^1.2.8",
"blake2": "^5.0.1",
"bluebird": "^3.7.2",
"bottleneck": "^2.19.5",
"cheerio": "^1.0.0-rc.2",
"config": "^1.30.0",
"date-fns": "^1.29.0",
"cheerio": "^1.2.0",
"config": "^4.4.2",
"date-fns": "^4.4.0",
"dist-exiftool": "^10.53.0",
"fluent-ffmpeg": "^2.1.2",
"fs-extra": "^5.0.0",
"js-yaml": "^3.12.0",
"jsdom": "^15.2.0",
"fluent-ffmpeg": "^2.1.3",
"fs-extra": "^11.4.0",
"js-yaml": "^5.2.3",
"mime": "^3.0.0",
"mime-types": "^2.1.18",
"node-cron": "^1.2.1",
"mime-types": "^3.0.2",
"node-cron": "^4.6.0",
"node-exiftool": "^2.3.0",
"node-fetch": "^2.1.2",
"object.omit": "^3.0.0",
"object.pick": "^1.3.0",
"sharp-phash": "^2.1.0",
"template-format": "^1.2.4",
"unprint": "^0.8.2",
"sharp-phash": "^2.2.0",
"template-format": "^1.2.5",
"unprint": "^0.19.20",
"url-pattern": "^1.0.3",
"winston": "^3.3.3",
"winston-daily-rotate-file": "^4.7.1",
"yargs": "^11.0.0",
"youtube-dl-exec": "^2.2.3"
"winston": "^3.19.0",
"winston-daily-rotate-file": "^5.0.0",
"yargs": "^17.7.2",
"youtube-dl-exec": "^3.1.12"
},
"devDependencies": {
"@babel/eslint-parser": "^7.19.1",
"eslint": "^8.34.0",
"@babel/eslint-parser": "^7.29.7",
"eslint": "^8.57.1",
"eslint-config-airbnb-base": "^15.0.0",
"eslint-plugin-import": "^2.27.5"
"eslint-plugin-import": "^2.32.0"
}
}

13
src/app.js
View File

@@ -140,8 +140,10 @@ function fetchSavePosts(userPosts, ep) {
}
async function initApp() {
let ep;
try {
const ep = new exiftool.ExiftoolProcess(exiftoolBin);
ep = new exiftool.ExiftoolProcess(exiftoolBin);
await ep.open();
if (args.fetch || args.fileDirect) {
@@ -154,8 +156,6 @@ async function initApp() {
await fetchSavePosts(userPosts, ep);
}
await ep.close();
if (args.watch) {
logger.info(`Watch-mode enabled, checking again for new posts according to crontab '${config.fetch.watch.schedule}'.`);
}
@@ -165,6 +165,13 @@ async function initApp() {
} else {
logger.error(error.message);
}
} finally {
// the exiftool child process keeps the event loop alive - if it's never closed
// (e.g. because an error above was thrown before we got to it), the process hangs
// forever after logging the error instead of actually exiting
if (ep) {
await ep.close().catch((error) => logger.error(`Could not close exiftool process: ${error.message}`));
}
}
}

1
src/archives/ip.js
View File

@@ -1,6 +1,5 @@
'use strict';
const fetch = require('node-fetch');
const $ = require('cheerio');
function findOnIp(username, page = 1, acc = []) {

5
src/cli.js
View File

@@ -124,6 +124,11 @@ function getArgs() {
type: 'boolean',
default: config.fetch.archives.search,
})
.option('debug', {
describe: 'Log full stack traces instead of just error messages',
type: 'boolean',
default: false,
})
.argv;
return {

73
src/fetch/content.js
View File

@@ -46,7 +46,7 @@ function selfPostToText(item, post) {
comments: curateComments(item.comments),
};
return yaml.safeDump(curatedPost);
return yaml.dump(curatedPost);
}
async function getBuffers(item, context) {
@@ -117,38 +117,51 @@ async function fetchSaveUserContent(user, ep, args) {
const items = await Promise.map(post.content.items, async (originalItem, index) => {
const item = { ...originalItem, index };
const buffers = await getBuffers(item, { post, host: post.host, headers: post.content.headers || item.headers });
// no buffers, ignore item
if (!buffers || buffers.length === 0) {
try {
const buffers = await getBuffers(item, { post, host: post.host, headers: post.content.headers || item.headers });
// no buffers, ignore item
if (!buffers || buffers.length === 0) {
return null;
}
item.hash = buffers[0].hash;
item.phash = buffers[0].phash;
// prevent duplicates
if (config.fetch.avoidDuplicates && hashes.has(buffers[0].hash) && !item.album) {
logger.verbose(`Ignoring duplicate file '${post.url}' (${post.permalink})`);
return {
...item,
ignored: true,
};
}
const filepath = getFilepath(item, post.content, post.host, post, user);
const sourcePaths = await save(filepath, buffers.map(({ buffer }) => buffer), item, post);
hashes.add(buffers[0].hash);
if (item.mux) {
await mux(filepath, sourcePaths, item);
}
await addMeta(filepath, item, post, user, ep);
return item;
} catch (error) {
// don't let one bad item (e.g. a post with a missing/invalid date) abort the
// entire run - log it with enough context to find it again and move on
logger.error(`Failed to fetch/save item ${index} for post '${post.id}' (${post.permalink}): ${error.message}`);
if (args.debug) {
logger.error(error.stack);
}
return null;
}
item.hash = buffers[0].hash;
item.phash = buffers[0].phash;
// prevent duplicates
if (config.fetch.avoidDuplicates && hashes.has(buffers[0].hash) && !item.album) {
logger.verbose(`Ignoring duplicate file '${post.url}' (${post.permalink})`);
return {
...item,
ignored: true,
};
}
const filepath = getFilepath(item, post.content, post.host, post, user);
const sourcePaths = await save(filepath, buffers.map(({ buffer }) => buffer), item, post);
hashes.add(buffers[0].hash);
if (item.mux) {
await mux(filepath, sourcePaths, item);
}
await addMeta(filepath, item, post, user, ep);
return item;
});
return {

26
src/interpolate.js
View File

@@ -9,6 +9,24 @@ const format = require('template-format');
const args = require('./cli')();
// date-fns throws a bare, contextless "Invalid time value" RangeError for any invalid date
// passed to format() - wrap it so a bad timestamp somewhere upstream (a post, item, user or
// album with a malformed created/createDate field) points back at its source instead.
// No date at all (e.g. profileDetails.js's item-less-of-a-datetime avatar interpolation) is a
// legitimate case, not a bug - that just interpolates to an empty string, same as any other
// unset field like title/description elsewhere in this function.
function safeFormat(date, dateFormat, label) {
if (date === null || date === undefined) {
return null;
}
if (!(date instanceof Date) || Number.isNaN(date.getTime())) {
throw new Error(`Cannot format ${label} date for filename - invalid datetime (${date})`);
}
return dateFns.format(date, dateFormat);
}
function interpolate(pattern, item = null, content = null, host = null, post = null, user = null, strip = true, dateFormat = config.library.dateFormat) {
const data = {
tags: {},
@@ -20,7 +38,7 @@ function interpolate(pattern, item = null, content = null, host = null, post = n
id: item.id,
title: item.title && item.title.slice(0, config.library.titleLength),
description: item.description,
date: dateFns.format(item.datetime, dateFormat),
date: safeFormat(item.datetime, dateFormat, 'item'),
index: item.index + config.library.indexOffset,
},
ext: item.type
@@ -49,7 +67,7 @@ function interpolate(pattern, item = null, content = null, host = null, post = n
id: content.album.id,
title: content.album.title && content.album.title.slice(0, config.library.titleLength),
description: content.album.description,
date: dateFns.format(content.album.datetime, dateFormat),
date: safeFormat(content.album.datetime, dateFormat, 'album'),
},
});
}
@@ -60,7 +78,7 @@ function interpolate(pattern, item = null, content = null, host = null, post = n
id: post.id,
title: post.title && post.title.slice(0, config.library.titleLength),
url: post.url,
date: dateFns.format(post.datetime, dateFormat),
date: safeFormat(post.datetime, dateFormat, 'post'),
index: post.index + config.library.indexOffset,
hash: post.hash,
score: post.score,
@@ -75,7 +93,7 @@ function interpolate(pattern, item = null, content = null, host = null, post = n
name: user.name,
username: user.name,
id: user.id,
created: dateFns.format(user.created, dateFormat),
created: safeFormat(user.created, dateFormat, 'user'),
},
tags: {
...data.tags,

1
src/methods/erome.js
View File

@@ -1,7 +1,6 @@
'use strict';
const config = require('config');
const fetch = require('node-fetch');
const cheerio = require('cheerio');
const logger = require('../logger')(__filename);

1
src/methods/eroshareAlbum.js
View File

@@ -1,7 +1,6 @@
'use strict';
const config = require('config');
const fetch = require('node-fetch');
async function eroshareAlbum(host) {
const res = await fetch(`https://web.archive.org/web/20170630040157im_/https://eroshare.com/${host.id}`);

1
src/methods/eroshareItem.js
View File

@@ -1,6 +1,5 @@
'use strict';
const fetch = require('node-fetch');
const cheerio = require('cheerio');
async function eroshareItem(host, post) {

1
src/methods/imgurAlbum.js
View File

@@ -1,7 +1,6 @@
'use strict';
const config = require('config');
const fetch = require('node-fetch');
const logger = require('../logger')(__filename);
const { fetchPredata } = require('./imgurImage');

1
src/methods/imgurImage.js
View File

@@ -1,7 +1,6 @@
'use strict';
const config = require('config');
const fetch = require('node-fetch');
async function fetchPredata() {
const data = {

41
src/methods/redditAlbum.js
View File

@@ -1,18 +1,32 @@
'use strict';
const mime = require('mime');
const bhttp = require('bhttp');
const { JSDOM } = require('jsdom');
// reddit gallery images: `s.u` (still) or `s.gif`/`s.mp4` (animated) - URLs come back
// HTML-entity-encoded (e.g. '&' instead of '&')
function resolveMediaUrl(media) {
const url = media?.s?.u || media?.s?.gif || media?.s?.mp4;
async function redditAlbum(host, post) {
const res = await bhttp.get(host.url);
return url ? url.replace(/&/g, '&') : null;
}
if (res.statusCode !== 200) {
throw new Error(res.body.toString());
}
async function redditAlbum(host, post, { reddit }) {
// fetched through the authenticated/throttled client, not a raw fetch()/bhttp request -
// reddit's anonymous-traffic bot challenge blocks plain unauthenticated .json requests
const data = await reddit.getSubmission(post.id);
const { document } = new JSDOM(res.body.toString(), { runScripts: 'dangerously' }).window;
const items = Array.from(document.querySelectorAll('li a'), el => el.href);
const items = (data.gallery_data?.items || [])
.filter((item) => !item.is_deleted)
.map((item) => {
const media = data.media_metadata?.[item.media_id];
const url = resolveMediaUrl(media);
return url && {
id: item.media_id,
url,
datetime: post.datetime,
type: media.m || 'image/jpeg',
};
})
.filter(Boolean);
return {
album: {
@@ -20,12 +34,7 @@ async function redditAlbum(host, post) {
url: host.url,
title: post.title,
},
items: items.map((url) => ({
id: new URL(url).pathname.match(/\/(.*).\w+$/)?.[1],
url,
datetime: post.datetime,
type: mime.getType(url) || 'image/jpeg',
})),
items,
};
}

51
src/methods/redditVideo.js
View File

@@ -1,23 +1,48 @@
'use strict';
const fetch = require('node-fetch');
// the DASH manifest lists the actual audio track filename - reddit has changed this naming
// scheme before (DASH_audio.mp4 -> /audio -> CMAF_AUDIO_128.mp4), so read it from the
// manifest rather than guessing a hardcoded suffix off the video URL again
function findAudioUrl(manifest, videoId) {
const audioSection = manifest.match(/<AdaptationSet[^>]*contentType="audio"[^>]*>([\s\S]*?)<\/AdaptationSet>/);
async function redditVideo(host, post) {
const res = await fetch(`${post.permalink}.json`);
const [{ data }] = await res.json();
if (!audioSection) {
return null;
}
const videoUrl = data.children[0].data.media.reddit_video.fallback_url;
const audioUrl = `${videoUrl.split('/').slice(0, -1).join('/')}/audio`;
const baseUrls = [...audioSection[1].matchAll(/<BaseURL>([^<]+)<\/BaseURL>/g)].map((match) => match[1]);
// representations are listed lowest to highest bandwidth - take the best one
const audioFile = baseUrls[baseUrls.length - 1];
const audioRes = await fetch(audioUrl, {
method: 'HEAD',
});
return audioFile ? `https://v.redd.it/${videoId}/${audioFile}` : null;
}
async function findAudio(video, videoId) {
if (!video.has_audio || !video.dash_url) {
return null;
}
const res = await fetch(video.dash_url);
if (!res.ok) {
return null;
}
return findAudioUrl(await res.text(), videoId);
}
async function redditVideo(host, post, { reddit }) {
// fetched through the authenticated/throttled client, not a raw fetch() - reddit's
// anonymous-traffic bot challenge blocks plain unauthenticated .json requests
const data = await reddit.getSubmission(post.id);
const video = data.media.reddit_video;
const videoId = post.host.id || post.id;
const item = {
album: null,
items: [{
id: post.host.id || post.id,
url: videoUrl,
id: videoId,
url: video.fallback_url,
title: post.title,
datetime: post.datetime,
type: 'video/mp4',
@@ -25,7 +50,9 @@ async function redditVideo(host, post) {
}],
};
if (audioRes.status === 200) {
const audioUrl = await findAudio(video, videoId);
if (audioUrl) {
item.items[0].mux = [audioUrl];
}

6
src/methods/redgifs.js
View File

@@ -1,6 +1,5 @@
'use strict';
const fetch = require('node-fetch');
const mime = require('mime');
// const unprint = require('unprint');
@@ -28,6 +27,11 @@ async function fetchPredata() {
}
function scrapeGallery(data, { predata }) {
if (!data.gifs.length) {
// Math.min() of an empty array is Infinity, which becomes an invalid date below
return null;
}
const oldestDate = Math.min(...data.gifs.map((gif) => gif.createDate));
const curated = {

2
src/methods/tube.js
View File

@@ -25,7 +25,7 @@ async function tube(host, post) {
title: data.fulltitle || data.title,
description: data.description,
type: `video/${data.ext}`,
datetime: dateFns.parse(data.upload_date, 'YYYYMMDD'),
datetime: dateFns.parse(data.upload_date, 'yyyyMMdd', new Date()),
original: data,
},
],

1
src/methods/vidbleAlbum.js
View File

@@ -1,7 +1,6 @@
'use strict';
const config = require('config');
const fetch = require('node-fetch');
const UrlPattern = require('url-pattern');
const cheerio = require('cheerio');
const mime = require('mime-types');

1
src/methods/vidbleImage.js
View File

@@ -1,6 +1,5 @@
'use strict';
const fetch = require('node-fetch');
const $ = require('cheerio');
const mime = require('mime-types');

1
src/methods/vidbleVideo.js
View File

@@ -1,6 +1,5 @@
'use strict';
const fetch = require('node-fetch');
const cheerio = require('cheerio');
async function vidbleVideo(host, post) {

1
src/reddit/client.js
View File

@@ -1,6 +1,5 @@
'use strict';
const fetch = require('node-fetch');
const Bottleneck = require('bottleneck');
const config = require('config');

2
src/save/mux.js
View File

@@ -18,7 +18,7 @@ async function mux(target, sources) {
resolve(stdout);
})
.on('error', () => reject)
.on('error', (error) => reject(error))
.save(target);
}).then(() => Promise.all(sources.map((source) => fs.remove(source))).then(() => {
logger.verbose(`Cleaned up temporary files for '${target}'`);

2
src/save/writeToIndex.js
View File

@@ -61,7 +61,7 @@ async function writeToIndex(posts, profilePaths, user, args) {
}
try {
const yamlIndex = yaml.safeDump(data, { skipInvalid: true });
const yamlIndex = yaml.dump(data, { skipInvalid: true });
const saved = await save(filepath, Buffer.from(yamlIndex, 'utf8'));
logger.info(`Saved index with ${posts.length} new posts for ${user.name}`);

2
src/sources/getIndex.js
View File

@@ -13,7 +13,7 @@ async function getIndex(user) {
try {
const indexFile = await fs.readFile(indexFilePath, 'utf8');
return yaml.safeLoad(indexFile, { json: true }); // allow duplicate keys
return yaml.load(indexFile, { json: true }); // allow duplicate keys
} catch (error) {
logger.info(`No index file found for '${user.name}' at '${indexFilePath}': ${error.message}`);