ripunzel/config/default.js

'use strict';

module.exports = {
    library: {
        base: {
            posts: 'output/{user.name}/{label}/',
            direct: 'output/{host.name}/{label}/',
        },
        posts: {
            image: '{base.posts}{post.date}{div}{tags.preview}{item.id}{div}{post.title}{ext}',
            video: '{base.posts}{post.date}{div}{tags.preview}{item.id}{div}{post.title}{ext}',
            text: '{base.posts}{post.date}{div}{tags.preview}{post.id}{div}{post.title}',
            album: {
                image: '{base.posts}{post.date}{div}{tags.preview}{album.id}{div}{post.title}/{item.index}{div}{item.id}{ext}',
                video: '{base.posts}{post.date}{div}{tags.preview}{album.id}{div}{post.title}/{item.index}{div}{item.id}{ext}',
            },
        },
        direct: {
            image: '{base.direct}{item.date}{div}{tags.preview}{item.id}{divs.item.title}{item.title}{ext}',
            video: '{base.direct}{item.date}{div}{tags.preview}{item.id}{divs.item.title}{item.title}{ext}',
            text: '{base.direct}{item.date}{div}{tags.preview}{item.id}{divs.item.title}{item.title}',
            album: {
                image: '{base.direct}{album.date}{div}{tags.preview}{album.id}{divs.album.title}{album.title}/{item.index}{div}{item.id}{ext}',
                video: '{base.direct}{album.date}{div}{tags.preview}{album.id}{divs.album.title}{album.title}/{item.index}{div}{item.id}{ext}',
            },
        },
        extractSingleAlbumItem: true,
        profile: {
            image: '{base.posts}{user.created}{div}profile{ext}',
            description: '{base.posts}{user.created}{div}profile ({tags.verified}{tags.verifiedEmail}{tags.gold}{tags.over18})',
            avoidAvatar: true,
        },
        index: {
            file: '{base.posts}index',
        },
        meta: {
            comment: '{item.description}',
        },
        tags: {
            extracted: 'extracted-',
            preview: 'preview-',
            verified: '✔',
            verifiedEmail: '✉',
            gold: '★',
            over18: '♥',
        },
        // date-fns v2+ token syntax (was 'YYYYMMDD' under date-fns v1)
        dateFormat: 'yyyyMMdd',
        truncate: {
            limit: 250,
            truncator: '...',
        },
        indexOffset: 1,
        divider: ' - ',
        slashSubstitute: '#',
    },
    fetch: {
        sort: 'new',
        limit: 1000,
        avoidDuplicates: true,
        retries: 3,
        concurrency: 10,
        // wall-clock ceiling (ms) for a single item-download attempt (src/fetch/item.js).
        // bhttp has no timeout of its own - a request that connects but then just stops
        // responding (a stalled/misbehaving proxy tunnel is a common cause) would otherwise
        // hang forever, and since the limiter's concurrency is capped, that one stuck request
        // blocks every other download queued behind it indefinitely. This bounds each attempt
        // so a stall fails (and retries, per `retries` above) instead of hanging forever.
        timeout: 60000,
        watch: {
            schedule: '*/30 * * * *',
        },
        // how long (ms) per-method predata (e.g. RedGifs' temporary API token) is reused before
        // being fetched again - matters for long --queue --wait runs
        predataTtl: 60 * 60 * 1000,
        archives: {
            search: false,
            preview: true,
            reddit: ['ip'],
            reupload: [],
        },
    },
    logger: {
        level: 'info',
    },
    limiter: {
        // default concurrency/interval for item/media downloads (src/fetch/item.js) - applies
        // to any host without its own entry in `hosts` below
        concurrency: 10,
        interval: 100,
        hosts: {
            // per-host overrides of `concurrency`/`interval` above, for hosts that can safely
            // be hit harder (most media CDNs) or need to be treated more cautiously than the
            // default. Hostnames are matched by suffix, same as proxy.hosts above - a bare
            // 'redgifs.com' also covers 'api.redgifs.com' and 'media.redgifs.com'. Unlike a
            // single shared limiter, each entry here runs independently: a host that's
            // currently stalling or being throttled doesn't block any other host's downloads.
            // 'redgifs.com': { concurrency: 20, interval: 50 },
        },
    },
    reddit: {
        // Netscape/Mozilla cookie-jar dump (string) of a logged-in reddit session, used to
        // get past reddit's anonymous-traffic bot challenge on www.reddit.com. Needs periodic
        // refreshing - src/reddit/client.js logs a warning once the soonest-expiring cookie
        // in here is within 48h of expiring.
        cookies: null,
        // fallback/simpler alternative to `cookies` above: a single pre-built `name=value; ...`
        // Cookie header string. Only used if `cookies` is not set.
        cookie: null,
        userAgent: 'ripunzel',
        throttle: {
            // requests per minute (and the minimum gap between two, in ms) for everything sent
            // to reddit itself - all of it is serialized through one limiter. reddit sends no
            // rate-limit headers for cookie sessions, so there's no documented ceiling; this is
            // public-endpoint access, not an approved API key, and getting flagged as a bot
            // defeats the point. 15/min held up in testing (2026-11), raise with care.
            reservoir: 15,
            minTime: 1500,
        },
    },
    queue: {
        // used by --queue when no filename is given
        file: 'queue',
        // with --queue --wait, how often (ms) an empty queue is checked for new users
        interval: 60 * 1000,
    },
    users: {
        // record of suspended/deleted accounts, and pruning them from the --file-users list - see
        // src/sources/accountStatus.js. Also run on its own with --cleanup.
        dead: {
            file: 'users_dead.yml',
            // also record and prune during regular archive runs (always done with --cleanup)
            duringArchive: true,
            // days an account must stay suspended/deleted (counted from the first run that saw
            // it, reddit exposes no date) before it's removed from the users file - until then
            // it's re-checked every run, so a lifted suspension is noticed. 'deleted' is really
            // "profile 404s", which reportedly includes shadowbanned accounts, hence not 0.
            retainDays: {
                suspended: 90,
                deleted: 90,
            },
        },
    },
    methods: {
        imgur: {
            clientId: null,
        },
    },
    proxy: {
        use: true,
        default: null,
        // if a request fails (including timing out - see fetch.timeout above) and is about to
        // be retried, retry it using whichever of proxied/direct is NOT what would normally
        // apply for that host - e.g. the proxy itself is down or being blocked by the target
        // site, or (the reverse) a host reachable directly is refusing unproxied traffic.
        // Off by default: for a privacy-sensitive proxy (e.g. Tor) silently falling back to a
        // direct connection on failure is very much not what you want.
        retryFlipped: false,
        hosts: {
            // hostnames are matched by suffix, so e.g. 'reddit.com' also covers
            // 'www.reddit.com', but not 'v.redd.it' or 'redd.it' - those need their own
            // entry, or fall back to `default`. A host entry can be:
            // - a proxy URL, to override `default` for that host
            // - `true`, to explicitly use `default` for that host
            // - `null`/`false`, to explicitly exclude that host from `default`
            // 'reddit.com': true,
            // 'v.redd.it': null,
            // 'imgur.com': 'socks5://127.0.0.1:9050',
        },
    },
};