boiii/traxxx
1
0
Fork 0
forked from DebaucheryLibrarian/traxxx
Code Issues Pull Requests Projects Releases Wiki Activity

Changed sites from argument query to group by network.

Browse Source
This commit is contained in:
ThePendulum
2020-06-15 03:58:35 +02:00
parent 79465d9634
commit 1907ce1e54
9 changed files with 65 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
src/entities.js
View File

@@ -14,16 +14,10 @@ function curateEntity(entity, includeParameters = false) {
url: entity.url,
description: entity.description,
slug: entity.slug,
independent: !!entity.parameters && entity.parameters.independent,
type: entity.type,
parameters: includeParameters ? entity.parameters : null,
network: {
id: entity.network_id,
name: entity.network_name,
description: entity.network_description,
slug: entity.network_slug,
url: entity.network_url,
parameters: includeParameters ? entity.network_parameters : null,
},
parent: entity.parent,
children: (entity.children || []).map(child => curateEntity(child)),
};
return curatedEntity;
@@ -33,56 +27,59 @@ async function curateEntities(entities, includeParameters) {
return Promise.all(entities.map(async entity => curateEntity(entity, includeParameters)));
}
async function findSiteByUrl(url) {
const { origin, hostname, pathname } = new URL(url);
// const domain = hostname.replace(/www.|tour./, '');
const dirUrl = `${origin}${pathname.split('/').slice(0, 2).join('/')}`; // allow for sites on URI directory
const site = await knex('sites')
.leftJoin('networks', 'sites.network_id', 'networks.id')
.select(
'sites.*',
'networks.name as network_name', 'networks.slug as network_slug', 'networks.url as network_url', 'networks.description as network_description', 'networks.parameters as network_parameters',
)
.where('sites.url', url)
.orWhere('sites.url', origin)
.orWhere('sites.url', origin.replace(/www\.|tour\./, ''))
.orWhere('sites.url', `https://www.${hostname}`)
.orWhere('sites.url', `http://www.${hostname}`)
.orWhere('sites.url', dirUrl)
// .orWhere('sites.url', 'like', `%${domain}`)
.first();
if (site) {
const curatedSite = curateSite(site, true, false);
return curatedSite;
}
return null;
}
async function fetchEntitiesFromArgv() {
async function fetchSitesFromArgv() {
const rawEntities = await knex.raw(`
WITH RECURSIVE temp AS (
SELECT id, parent_id, name, slug, type FROM entities WHERE slug IN (?)
UNION
SELECT entities.id, entities.parent_id, entities.name, entities.slug, entities.type FROM entities
INNER JOIN temp ON temp.id = entities.parent_id
) SELECT * FROM temp;
`, argv.sites || argv.networks || argv.entities);
console.log(rawEntities.rows);
SELECT
id, parent_id, name, slug, type, url, description, parameters
FROM
entities
WHERE
slug = ANY(?) AND entities.type = 1
UNION ALL
SELECT
entities.id, entities.parent_id, entities.name, entities.slug, entities.type, entities.url, entities.description, entities.parameters
FROM
entities
INNER JOIN
temp ON temp.id = entities.parent_id
)
SELECT
entities.*, row_to_json(parents) as parent, json_agg(temp) as children
FROM
temp
LEFT JOIN
entities ON entities.id = temp.parent_id
LEFT JOIN
entities AS parents ON parents.id = entities.parent_id
WHERE
temp.type = 2
GROUP BY
temp.parent_id, entities.id, entities.name, parents.id
UNION ALL
SELECT
entities.*, row_to_json(parents) as parent, json_build_array(row_to_json(children))
FROM
entities AS children
LEFT JOIN
entities ON entities.id = children.parent_id
LEFT JOIN
entities AS parents ON parents.id = entities.parent_id
WHERE
children.slug = ANY(?) AND children.type = 2
GROUP BY
entities.id, parents.id, children.id;
`, [argv.networks || [], argv.sites || []]);
const curatedEntities = await curateEntities(rawEntities.rows, true);
logger.info(`Found ${curatedEntities.length} entities in database`);
console.log(curatedEntities);
console.log(rawEntities.rows);
return curatedEntities;
}
async function fetchEntitiesFromConfig() {
async function fetchSitesFromConfig() {
const rawSites = await knex('entities')
.select('entities.*')
.leftJoin('entities as entities_parents', 'entities_parents.id', 'entities.id')
@@ -95,18 +92,18 @@ async function fetchEntitiesFromConfig() {
builder.whereIn('entities.slug', config.exclude || []);
});
const curatedSites = await curateSites(rawSites, true);
logger.info(`Found ${curatedSites.length} sites in database`);
const curatedSites = await curateEntities(rawSites, true);
logger.info(`Found ${curatedSites.length} entities in database`);
return curatedSites;
}
async function fetchIncludedEntities() {
if (argv.networks || argv.sites) {
return fetchEntitiesFromArgv();
return fetchSitesFromArgv();
}
return fetchEntitiesFromConfig();
return fetchSitesFromConfig();
}
async function fetchSites(queryObject) {
@@ -119,7 +116,7 @@ async function fetchSites(queryObject) {
.leftJoin('networks', 'sites.network_id', 'networks.id')
.limit(100);
return curateSites(sites);
return curateEntities(sites);
}
async function fetchSitesFromReleases() {
@@ -129,7 +126,7 @@ async function fetchSitesFromReleases() {
.groupBy('sites.id')
.limit(100);
return curateSites(sites);
return curateEntities(sites);
}
module.exports = {
@@ -137,8 +134,7 @@ module.exports = {
curateEntities,
fetchIncludedEntities,
fetchSites,
fetchEntitiesFromConfig,
fetchEntitiesFromArgv,
fetchSitesFromConfig,
fetchSitesFromArgv,
fetchSitesFromReleases,
findSiteByUrl,
};

2
src/scrapers/hush.js
View File

@@ -108,7 +108,7 @@ function scrapeAllT1(scenes, site, accSiteReleases) {
release.entryId = deriveEntryId(release);
if (site.parameters?.accFilter && accSiteReleases?.map(accRelease => accRelease.entryId).includes(release.entryId)) {
// filter out releases that were already scraped from a categorized site
// filter out releases that were already scraped from a categorized site, requires sequential site scraping
return null;
}

8
src/updates.js
View File

@@ -8,7 +8,7 @@ const logger = require('./logger')(__filename);
const knex = require('./knex');
const include = require('./utils/argv-include')(argv);
const scrapers = require('./scrapers/scrapers');
const { fetchEntitiesFromArgv, fetchEntitiesFromConfig } = require('./entities');
const { fetchSitesFromArgv, fetchSitesFromConfig } = require('./entities');
const afterDate = (() => {
if (/\d{2,4}-\d{2}-\d{2,4}/.test(argv.after)) {
@@ -219,8 +219,10 @@ async function scrapeNetworkParallel(network) {
async function fetchUpdates() {
const includedNetworks = argv.sites || argv.networks || argv.from
? await fetchEntitiesFromArgv()
: await fetchEntitiesFromConfig();
? await fetchSitesFromArgv()
: await fetchSitesFromConfig();
// console.log('included', includedNetworks);
const scrapedNetworks = await Promise.map(
includedNetworks,