'use strict'; module.exports = { library: { base: { posts: 'output/{user.name}/{label}/', direct: 'output/{host.name}/{label}/', }, posts: { image: '{base.posts}{post.date}{div}{tags.preview}{item.id}{div}{post.title}{ext}', video: '{base.posts}{post.date}{div}{tags.preview}{item.id}{div}{post.title}{ext}', text: '{base.posts}{post.date}{div}{tags.preview}{post.id}{div}{post.title}', album: { image: '{base.posts}{post.date}{div}{tags.preview}{album.id}{div}{post.title}/{item.index}{div}{item.id}{ext}', video: '{base.posts}{post.date}{div}{tags.preview}{album.id}{div}{post.title}/{item.index}{div}{item.id}{ext}', }, }, direct: { image: '{base.direct}{item.date}{div}{tags.preview}{item.id}{divs.item.title}{item.title}{ext}', video: '{base.direct}{item.date}{div}{tags.preview}{item.id}{divs.item.title}{item.title}{ext}', text: '{base.direct}{item.date}{div}{tags.preview}{item.id}{divs.item.title}{item.title}', album: { image: '{base.direct}{album.date}{div}{tags.preview}{album.id}{divs.album.title}{album.title}/{item.index}{div}{item.id}{ext}', video: '{base.direct}{album.date}{div}{tags.preview}{album.id}{divs.album.title}{album.title}/{item.index}{div}{item.id}{ext}', }, }, extractSingleAlbumItem: true, profile: { image: '{base.posts}{user.created}{div}profile{ext}', description: '{base.posts}{user.created}{div}profile ({tags.verified}{tags.verifiedEmail}{tags.gold}{tags.over18})', avoidAvatar: true, }, index: { file: '{base.posts}index', }, meta: { comment: '{item.description}', }, tags: { extracted: 'extracted-', preview: 'preview-', verified: '✔', verifiedEmail: '✉', gold: '★', over18: '♥', }, // date-fns v2+ token syntax (was 'YYYYMMDD' under date-fns v1) dateFormat: 'yyyyMMdd', truncate: { limit: 250, truncator: '...', }, indexOffset: 1, divider: ' - ', slashSubstitute: '#', }, fetch: { sort: 'new', limit: 1000, avoidDuplicates: true, retries: 3, concurrency: 10, // wall-clock ceiling (ms) for a single item-download attempt (src/fetch/item.js). // bhttp has no timeout of its own - a request that connects but then just stops // responding (a stalled/misbehaving proxy tunnel is a common cause) would otherwise // hang forever, and since the limiter's concurrency is capped, that one stuck request // blocks every other download queued behind it indefinitely. This bounds each attempt // so a stall fails (and retries, per `retries` above) instead of hanging forever. timeout: 60000, watch: { schedule: '*/30 * * * *', }, archives: { search: false, preview: true, reddit: ['ip'], reupload: [], }, }, logger: { level: 'info', }, limiter: { concurrency: 10, interval: 100, }, reddit: { // Netscape/Mozilla cookie-jar dump (string) of a logged-in reddit session, used to // get past reddit's anonymous-traffic bot challenge on www.reddit.com. Needs periodic // refreshing - src/reddit/client.js logs a warning once the soonest-expiring cookie // in here is within 48h of expiring. cookies: null, // fallback/simpler alternative to `cookies` above: a single pre-built `name=value; ...` // Cookie header string. Only used if `cookies` is not set. cookie: null, userAgent: 'ripunzel', throttle: { // deliberately conservative - this is unauthenticated-looking public access, // not an approved API key, getting flagged as a bot defeats the point reservoir: 6, minTime: 1500, }, }, methods: { imgur: { clientId: null, }, }, proxy: { use: true, default: null, // if a request fails (including timing out - see fetch.timeout above) and is about to // be retried, retry it using whichever of proxied/direct is NOT what would normally // apply for that host - e.g. the proxy itself is down or being blocked by the target // site, or (the reverse) a host reachable directly is refusing unproxied traffic. // Off by default: for a privacy-sensitive proxy (e.g. Tor) silently falling back to a // direct connection on failure is very much not what you want. retryFlipped: false, hosts: { // hostnames are matched by suffix, so e.g. 'reddit.com' also covers // 'www.reddit.com', but not 'v.redd.it' or 'redd.it' - those need their own // entry, or fall back to `default`. A host entry can be: // - a proxy URL, to override `default` for that host // - `true`, to explicitly use `default` for that host // - `null`/`false`, to explicitly exclude that host from `default` // 'reddit.com': true, // 'v.redd.it': null, // 'imgur.com': 'socks5://127.0.0.1:9050', }, }, };