Fixed Porn Pros scraper. Added various Score site logos.

This commit is contained in:
2020-02-18 16:00:36 +01:00
parent cabae4989e
commit 40bf476ea6
99 changed files with 483 additions and 316 deletions

37
src/utils/scorelogos.js Normal file
View File

@@ -0,0 +1,37 @@
'use strict';
const Promise = require('bluebird');
const bhttp = require('bhttp');
const fs = require('fs-extra');
const knex = require('../knex');
async function init() {
const sites = await knex('sites')
.select('networks.name', 'sites.slug')
.join('networks', 'networks.id', 'sites.network_id')
.where('networks.slug', 'score');
await Promise.map(sites, async (site) => {
const url = `https://cdn77.scoreuniverse.com/${site.slug}/images/logo.png`;
console.log(url);
const res = await bhttp.get(url, {
responseTimeout: 5000,
});
if (res.statusCode === 200) {
console.log(`Saving logo for ${site.slug}`);
await fs.writeFile(`./score/${site.slug}.png`, res.body);
}
console.log(`No logo found for ${site.slug}`);
}, {
concurrency: 10,
});
knex.destroy();
}
init();