|
|
|
@ -231,44 +231,50 @@ function groupItems(items) {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function storeMedia(sources, domain, role) {
|
|
|
|
|
const presentSources = sources.filter(Boolean);
|
|
|
|
|
try {
|
|
|
|
|
const presentSources = sources.filter(Boolean);
|
|
|
|
|
|
|
|
|
|
if (presentSources.length === 0) {
|
|
|
|
|
return {};
|
|
|
|
|
if (presentSources.length === 0) {
|
|
|
|
|
return {};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// find source duplicates that don't need to be re-downloaded or re-saved
|
|
|
|
|
const existingSourceItems = await knex('media').whereIn('source', presentSources.flat().map(source => source.src || source));
|
|
|
|
|
const { source: existingSourceItemsBySource, hash: existingSourceItemsByHash } = groupItems(existingSourceItems);
|
|
|
|
|
|
|
|
|
|
// download media items from new sources
|
|
|
|
|
const fetchedItems = await fetchItems(presentSources, existingSourceItemsBySource, domain, role);
|
|
|
|
|
const { hash: fetchedItemsByHash } = groupItems(fetchedItems);
|
|
|
|
|
|
|
|
|
|
// find hash duplicates that don't need to be re-saved
|
|
|
|
|
const uniqueFetchedItems = Object.values(fetchedItemsByHash);
|
|
|
|
|
const existingHashItems = await knex('media').whereIn('hash', uniqueFetchedItems.map(item => item.hash));
|
|
|
|
|
const { hash: existingHashItemsByHash } = groupItems(existingHashItems);
|
|
|
|
|
|
|
|
|
|
// save new items to disk
|
|
|
|
|
const newItems = uniqueFetchedItems.filter(item => !existingHashItemsByHash[item.hash]);
|
|
|
|
|
const savedItems = await saveItems(newItems, domain, role);
|
|
|
|
|
|
|
|
|
|
// store new items in database
|
|
|
|
|
const curatedItemEntries = curateItemEntries(savedItems);
|
|
|
|
|
const storedItems = await knex('media').insert(curatedItemEntries).returning('*');
|
|
|
|
|
const { hash: storedItemsByHash } = groupItems(Array.isArray(storedItems) ? storedItems : []);
|
|
|
|
|
|
|
|
|
|
// accumulate existing and new items by source to be mapped onto releases
|
|
|
|
|
const itemsByHash = { ...existingSourceItemsByHash, ...existingHashItemsByHash, ...storedItemsByHash };
|
|
|
|
|
const itemsBySource = {
|
|
|
|
|
...existingSourceItemsBySource,
|
|
|
|
|
...fetchedItems.reduce((acc, item) => ({ ...acc, [item.source]: itemsByHash[item.hash] }), {}),
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
logger.info(`Stored ${fetchedItems.length} new ${domain} ${role}s`);
|
|
|
|
|
|
|
|
|
|
return itemsBySource;
|
|
|
|
|
} catch (error) {
|
|
|
|
|
logger.error(`Failed to store ${domain} ${role} batch: ${error.message}`);
|
|
|
|
|
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// find source duplicates that don't need to be re-downloaded or re-saved
|
|
|
|
|
const existingSourceItems = await knex('media').whereIn('source', presentSources.flat().map(source => source.src || source));
|
|
|
|
|
const { source: existingSourceItemsBySource, hash: existingSourceItemsByHash } = groupItems(existingSourceItems);
|
|
|
|
|
|
|
|
|
|
// download media items from new sources
|
|
|
|
|
const fetchedItems = await fetchItems(presentSources, existingSourceItemsBySource, domain, role);
|
|
|
|
|
const { hash: fetchedItemsByHash } = groupItems(fetchedItems);
|
|
|
|
|
|
|
|
|
|
// find hash duplicates that don't need to be re-saved
|
|
|
|
|
const uniqueFetchedItems = Object.values(fetchedItemsByHash);
|
|
|
|
|
const existingHashItems = await knex('media').whereIn('hash', uniqueFetchedItems.map(item => item.hash));
|
|
|
|
|
const { hash: existingHashItemsByHash } = groupItems(existingHashItems);
|
|
|
|
|
|
|
|
|
|
// save new items to disk
|
|
|
|
|
const newItems = uniqueFetchedItems.filter(item => !existingHashItemsByHash[item.hash]);
|
|
|
|
|
const savedItems = await saveItems(newItems, domain, role);
|
|
|
|
|
|
|
|
|
|
// store new items in database
|
|
|
|
|
const curatedItemEntries = curateItemEntries(savedItems);
|
|
|
|
|
const storedItems = await knex('media').insert(curatedItemEntries).returning('*');
|
|
|
|
|
const { hash: storedItemsByHash } = groupItems(Array.isArray(storedItems) ? storedItems : []);
|
|
|
|
|
|
|
|
|
|
// accumulate existing and new items by source to be mapped onto releases
|
|
|
|
|
const itemsByHash = { ...existingSourceItemsByHash, ...existingHashItemsByHash, ...storedItemsByHash };
|
|
|
|
|
const itemsBySource = {
|
|
|
|
|
...existingSourceItemsBySource,
|
|
|
|
|
...fetchedItems.reduce((acc, item) => ({ ...acc, [item.source]: itemsByHash[item.hash] }), {}),
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
logger.info(`Stored ${fetchedItems.length} new ${domain} ${role}s`);
|
|
|
|
|
|
|
|
|
|
return itemsBySource;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function extractPrimaryItem(associations, targetId, role, primaryRole, primaryItemsByTargetId) {
|
|
|
|
@ -302,7 +308,7 @@ function associateTargetMedia(targetId, sources, mediaBySource, domain, role, pr
|
|
|
|
|
})
|
|
|
|
|
.filter(Boolean);
|
|
|
|
|
|
|
|
|
|
logger.info(`Associating ${associations.length} ${role}s to ${domain} ${targetId}`);
|
|
|
|
|
logger.silly(`Associating ${associations.length} ${role}s to ${domain} ${targetId}`);
|
|
|
|
|
|
|
|
|
|
return extractPrimaryItem(associations, targetId, role, primaryRole, primaryItemsByTargetId);
|
|
|
|
|
}
|
|
|
|
@ -316,6 +322,9 @@ async function associateMedia(sourcesByTargetId, mediaBySource, domain, role, pr
|
|
|
|
|
const associations = associationsPerTarget.map(association => association[role]).flat().filter(Boolean);
|
|
|
|
|
const primaryAssociations = associationsPerTarget.map(association => association[primaryRole]).filter(Boolean);
|
|
|
|
|
|
|
|
|
|
logger.info(`Associated ${associations.length} ${role}s to ${domain}s`);
|
|
|
|
|
logger.info(`Associated ${primaryAssociations.length} extracted ${primaryRole}s to ${domain}s`);
|
|
|
|
|
|
|
|
|
|
return Promise.all([
|
|
|
|
|
(associations.length > 0 && knex.raw(`${knex(`${domain}s_${role}s`).insert(associations).toString()} ON CONFLICT DO NOTHING`)),
|
|
|
|
|
(primaryAssociations.length > 0 && knex.raw(`${knex(`${domain}s_${primaryRole}s`).insert(primaryAssociations).toString()} ON CONFLICT DO NOTHING`)),
|
|
|
|
|