Improved release storage module. Added new tags module. Added movie scraping.

This commit is contained in:
2020-03-22 03:50:24 +01:00
parent d765543b30
commit fdb2b132f6
24 changed files with 343 additions and 141 deletions

View File

@@ -52,11 +52,11 @@ queue.define('http', async ({
const reqOptions = {
headers: {
...(options.defaultHeaders !== false && defaultHeaders),
...headers,
...defaultHeaders,
},
...options,
...defaultOptions,
...options,
...(options.timeout && { responseTimeout: options.timeout }),
};

View File

@@ -288,9 +288,7 @@ function extractAll(htmlValue, selector) {
}
async function get(urlValue, selector, headers, options, queryAll = false) {
const res = await http.get(urlValue, {
headers,
});
const res = await http.get(urlValue, headers);
if (res.statusCode === 200) {
const item = queryAll

View File

@@ -4,6 +4,10 @@ function slugify(string, delimiter = '-', {
encode = false,
limit = 1000,
} = {}) {
if (!string) {
return '';
}
const slugComponents = string.trim().toLowerCase().match(/\w+/g);
if (!slugComponents) {