Switched to tabs. Adding missing actor entries when scraping actors, with batch ID.

This commit is contained in:
2020-05-14 04:26:05 +02:00
parent f1eb29c713
commit 11eb66f834
178 changed files with 16594 additions and 16929 deletions

View File

@@ -4,33 +4,33 @@ const Promise = require('bluebird');
const knex = require('../knex');
async function listSites() {
const [networks, allSites] = await Promise.all([
knex('networks').orderBy('name'),
knex('sites').orderBy('name'),
]);
const [networks, allSites] = await Promise.all([
knex('networks').orderBy('name'),
knex('sites').orderBy('name'),
]);
await Promise.each(networks, async (network) => {
console.log(`* **${network.name}**`);
await Promise.each(networks, async (network) => {
console.log(`* **${network.name}**`);
const sites = await knex('sites')
.where({ network_id: network.id })
.orderBy('name');
const sites = await knex('sites')
.where({ network_id: network.id })
.orderBy('name');
if (sites.length === 1 && sites[0].name === network.name) {
return;
}
if (sites.length === 1 && sites[0].name === network.name) {
return;
}
sites.forEach((site) => {
const rkSpecial = network.id === 'realitykings'
sites.forEach((site) => {
const rkSpecial = network.id === 'realitykings'
&& (new URL(site.url).hostname === 'www.realitykings.com'
|| (site.parameters?.altLayout))
? '\\*' : ''; // Reality Kings alt layout sites do not support scene fetch by URL
? '\\*' : ''; // Reality Kings alt layout sites do not support scene fetch by URL
console.log(` * ${site.name}${rkSpecial}`);
});
});
console.log(` * ${site.name}${rkSpecial}`);
});
});
console.log(`${networks.length} networks with ${allSites.length} sites total`);
console.log(`${networks.length} networks with ${allSites.length} sites total`);
}
listSites();