Ignoring empty interpolated metadata. Added URL query wildcard to imgur and reddit images. Using host ID rather than URL to improve duplicate detection.
This commit is contained in:
parent
ad78bac5de
commit
5a3d2dd030
|
@ -7,7 +7,9 @@ function curateSubmissions(submissions) {
|
||||||
const processed = new Set();
|
const processed = new Set();
|
||||||
|
|
||||||
return submissions.reduce((acc, submission, index) => {
|
return submissions.reduce((acc, submission, index) => {
|
||||||
if(config.fetch.avoidDuplicates && processed.has(submission.url)) {
|
const host = dissectLink(submission.url);
|
||||||
|
|
||||||
|
if(config.fetch.avoidDuplicates && processed.has(host.id)) {
|
||||||
console.log('\x1b[33m%s\x1b[0m', `Ignoring cross-post or repost '${submission.title}' - ${submission.url}`);
|
console.log('\x1b[33m%s\x1b[0m', `Ignoring cross-post or repost '${submission.title}' - ${submission.url}`);
|
||||||
|
|
||||||
return acc;
|
return acc;
|
||||||
|
@ -23,10 +25,10 @@ function curateSubmissions(submissions) {
|
||||||
url: submission.url,
|
url: submission.url,
|
||||||
datetime: new Date(submission.created_utc * 1000),
|
datetime: new Date(submission.created_utc * 1000),
|
||||||
subreddit: submission.subreddit.display_name,
|
subreddit: submission.subreddit.display_name,
|
||||||
host: dissectLink(submission.url)
|
host
|
||||||
};
|
};
|
||||||
|
|
||||||
processed.add(submission.url);
|
processed.add(host.id);
|
||||||
|
|
||||||
return acc.concat(curatedSubmission);
|
return acc.concat(curatedSubmission);
|
||||||
}, []);
|
}, []);
|
||||||
|
|
|
@ -13,11 +13,11 @@ const hosts = [{
|
||||||
}, {
|
}, {
|
||||||
method: 'reddit',
|
method: 'reddit',
|
||||||
label: 'reddit',
|
label: 'reddit',
|
||||||
pattern: new urlPattern('https\\://i.reddituploads.com/:id?(fit=:fit)(&h=:height)(&w=:width)(&s=:signature)')
|
pattern: new urlPattern('https\\://i.reddituploads.com/:id(?*)')
|
||||||
}, {
|
}, {
|
||||||
method: 'imgurImage',
|
method: 'imgurImage',
|
||||||
label: 'imgur',
|
label: 'imgur',
|
||||||
pattern: new urlPattern('http(s)\\://(i.)imgur.com/:id(.:ext)(?:num)')
|
pattern: new urlPattern('http(s)\\://(i.)imgur.com/:id(.:ext)(?*)')
|
||||||
}, {
|
}, {
|
||||||
method: 'imgurAlbum',
|
method: 'imgurAlbum',
|
||||||
label: 'imgur',
|
label: 'imgur',
|
||||||
|
|
|
@ -43,7 +43,11 @@ module.exports = function(posts, user) {
|
||||||
return save(filepath, item.stream);
|
return save(filepath, item.stream);
|
||||||
}).then(() => {
|
}).then(() => {
|
||||||
const interpolatedMeta = Object.entries(config.library.meta).reduce((acc, [key, value]) => {
|
const interpolatedMeta = Object.entries(config.library.meta).reduce((acc, [key, value]) => {
|
||||||
acc[key] = interpolate(value, user, post, item);
|
const interpolatedValue = interpolate(value, user, post, item);
|
||||||
|
|
||||||
|
if(interpolatedValue) {
|
||||||
|
acc[key] = interpolatedValue;
|
||||||
|
}
|
||||||
|
|
||||||
return acc;
|
return acc;
|
||||||
}, {});
|
}, {});
|
||||||
|
@ -58,7 +62,6 @@ module.exports = function(posts, user) {
|
||||||
});
|
});
|
||||||
}));
|
}));
|
||||||
}).then(() => {
|
}).then(() => {
|
||||||
console.log('closing...');
|
|
||||||
return ep.close();
|
return ep.close();
|
||||||
}).catch(error => {
|
}).catch(error => {
|
||||||
return ep.close();
|
return ep.close();
|
||||||
|
|
Loading…
Reference in New Issue