DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Including all children of included networks, separated included children into dedicated property.

This commit is contained in:
DebaucheryLibrarian 2020-11-26 04:26:52 +01:00
parent 1b407254a7
commit 4e559f63e3
4 changed files with 16 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
migrations/20190325001339_releases.js
View File

@ -156,7 +156,8 @@ exports.up = knex => Promise.resolve()
table.integer('parent_id', 12) table.integer('parent_id', 12)
.references('id') .references('id')
.inTable('entities'); .inTable('entities')
.index();
table.text('name'); table.text('name');
table.text('slug', 32); table.text('slug', 32);

22
src/entities.js
View File

@ -39,14 +39,11 @@ function curateEntity(entity, includeParameters = false) {
}, includeParameters)); }, includeParameters));
} }
if (entity.siblings) { if (entity.included_children) {
curatedEntity.parent = { curatedEntity.includedChildren = entity.included_children.map(child => curateEntity({
...curatedEntity.parent, ...child,
children: entity.siblings.map(sibling => curateEntity({ parent: curatedEntity.id ? curatedEntity : null,
...sibling, }, includeParameters));
parent: curatedEntity.parent,
}, includeParameters)),
};
} }
if (entity.tags) { if (entity.tags) {
@ -119,7 +116,11 @@ async function fetchIncludedEntities() {
) )
/* select recursive channels as children of networks */ /* select recursive channels as children of networks */
SELECT SELECT
parents.*, json_agg(included_entities ORDER BY included_entities.id) as children parents.*,
json_agg(included_entities ORDER BY included_entities.id) included_children,
(SELECT json_agg(children)
FROM entities AS children
WHERE children.parent_id = parents.id) children
FROM FROM
included_entities included_entities
LEFT JOIN LEFT JOIN
@ -130,9 +131,6 @@ async function fetchIncludedEntities() {
parents.id; parents.id;
`, include); `, include);
// console.log(rawNetworks.rows[0]);
// console.log(rawNetworks.toString());
const curatedNetworks = rawNetworks.rows.map(entity => curateEntity(entity, true)); const curatedNetworks = rawNetworks.rows.map(entity => curateEntity(entity, true));
return curatedNetworks; return curatedNetworks;

5
src/scrapers/littlecapricedreams.js
View File

@ -6,8 +6,6 @@ const slugify = require('../utils/slugify');
function matchChannel(release, channel) { function matchChannel(release, channel) {
const series = channel.children || channel.parent.children; const series = channel.children || channel.parent.children;
// console.log(series?.length, release.url, channel.name);
const serieNames = series.reduce((acc, serie) => ({ const serieNames = series.reduce((acc, serie) => ({
...acc, ...acc,
[serie.name]: serie, [serie.name]: serie,
@ -19,7 +17,8 @@ function matchChannel(release, channel) {
const serieName = release.title.match(new RegExp(Object.keys(serieNames).join('|'), 'i'))?.[0]; const serieName = release.title.match(new RegExp(Object.keys(serieNames).join('|'), 'i'))?.[0];
const serie = serieName && serieNames[slugify(serieName, '')]; const serie = serieName && serieNames[slugify(serieName, '')];
console.log(release.title, serieName); console.log(release.title);
console.log(serieName);
if (serie) { if (serie) {
return { return {

4
src/updates.js
View File

@ -230,7 +230,7 @@ async function scrapeChannel(channelEntity, accNetworkReleases) {
async function scrapeNetworkSequential(networkEntity) { async function scrapeNetworkSequential(networkEntity) {
const releases = await Promise.reduce( const releases = await Promise.reduce(
networkEntity.children, networkEntity.includedChildren,
async (chain, channelEntity) => { async (chain, channelEntity) => {
const accNetworkReleases = await chain; const accNetworkReleases = await chain;
const { uniqueReleases, duplicateReleases } = await scrapeChannel(channelEntity, accNetworkReleases); const { uniqueReleases, duplicateReleases } = await scrapeChannel(channelEntity, accNetworkReleases);
@ -248,7 +248,7 @@ async function scrapeNetworkSequential(networkEntity) {
async function scrapeNetworkParallel(networkEntity) { async function scrapeNetworkParallel(networkEntity) {
return Promise.map( return Promise.map(
networkEntity.children, networkEntity.includedChildren,
async (channelEntity) => { async (channelEntity) => {
const { uniqueReleases } = await scrapeChannel(channelEntity, networkEntity); const { uniqueReleases } = await scrapeChannel(channelEntity, networkEntity);