Fixed countries seed file. Updated MOFOS scraper. Improved Reality Kings scraper. Limiting photos for XEmpire scraper.
This commit is contained in:
21
src/utils/mofos.js
Normal file
21
src/utils/mofos.js
Normal file
@@ -0,0 +1,21 @@
|
||||
'use strict';
|
||||
|
||||
const Promise = require('bluebird');
|
||||
const bhttp = require('bhttp');
|
||||
|
||||
const knex = require('../knex');
|
||||
|
||||
async function run() {
|
||||
const network = await knex('networks').where('slug', 'mofos').first();
|
||||
const sites = await knex('sites').where('network_id', network.id);
|
||||
|
||||
await Promise.map(sites, async (site) => {
|
||||
const res = await bhttp.get(site.url);
|
||||
|
||||
console.log(site.url, res.statusCode);
|
||||
}, {
|
||||
concurrency: 5,
|
||||
});
|
||||
}
|
||||
|
||||
run();
|
||||
Reference in New Issue
Block a user