DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 37 Pull Requests 1 Releases Wiki Activity

Catching media failures per batch. Refined teaser logging.

This commit is contained in:
ThePendulum 2020-02-20 22:27:00 +01:00
parent 278246a343
commit 7ac5a8e08c
3 changed files with 53 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
assets/css/style.scss
View File

@ -43,7 +43,7 @@ body {
display: flex;
justify-content: center;
align-items: center;
padding: .5rem 0;
padding: .5rem .25rem;
font-weight: bold;
font-size: .9rem;
cursor: pointer;

83
src/media.js
View File

@ -231,44 +231,50 @@ function groupItems(items) {
}
async function storeMedia(sources, domain, role) {
const presentSources = sources.filter(Boolean);
try {
const presentSources = sources.filter(Boolean);
if (presentSources.length === 0) {
return {};
if (presentSources.length === 0) {
return {};
}
// find source duplicates that don't need to be re-downloaded or re-saved
const existingSourceItems = await knex('media').whereIn('source', presentSources.flat().map(source => source.src || source));
const { source: existingSourceItemsBySource, hash: existingSourceItemsByHash } = groupItems(existingSourceItems);
// download media items from new sources
const fetchedItems = await fetchItems(presentSources, existingSourceItemsBySource, domain, role);
const { hash: fetchedItemsByHash } = groupItems(fetchedItems);
// find hash duplicates that don't need to be re-saved
const uniqueFetchedItems = Object.values(fetchedItemsByHash);
const existingHashItems = await knex('media').whereIn('hash', uniqueFetchedItems.map(item => item.hash));
const { hash: existingHashItemsByHash } = groupItems(existingHashItems);
// save new items to disk
const newItems = uniqueFetchedItems.filter(item => !existingHashItemsByHash[item.hash]);
const savedItems = await saveItems(newItems, domain, role);
// store new items in database
const curatedItemEntries = curateItemEntries(savedItems);
const storedItems = await knex('media').insert(curatedItemEntries).returning('*');
const { hash: storedItemsByHash } = groupItems(Array.isArray(storedItems) ? storedItems : []);
// accumulate existing and new items by source to be mapped onto releases
const itemsByHash = { ...existingSourceItemsByHash, ...existingHashItemsByHash, ...storedItemsByHash };
const itemsBySource = {
...existingSourceItemsBySource,
...fetchedItems.reduce((acc, item) => ({ ...acc, [item.source]: itemsByHash[item.hash] }), {}),
};
logger.info(`Stored ${fetchedItems.length} new ${domain} ${role}s`);
return itemsBySource;
} catch (error) {
logger.error(`Failed to store ${domain} ${role} batch: ${error.message}`);
return null;
}
// find source duplicates that don't need to be re-downloaded or re-saved
const existingSourceItems = await knex('media').whereIn('source', presentSources.flat().map(source => source.src || source));
const { source: existingSourceItemsBySource, hash: existingSourceItemsByHash } = groupItems(existingSourceItems);
// download media items from new sources
const fetchedItems = await fetchItems(presentSources, existingSourceItemsBySource, domain, role);
const { hash: fetchedItemsByHash } = groupItems(fetchedItems);
// find hash duplicates that don't need to be re-saved
const uniqueFetchedItems = Object.values(fetchedItemsByHash);
const existingHashItems = await knex('media').whereIn('hash', uniqueFetchedItems.map(item => item.hash));
const { hash: existingHashItemsByHash } = groupItems(existingHashItems);
// save new items to disk
const newItems = uniqueFetchedItems.filter(item => !existingHashItemsByHash[item.hash]);
const savedItems = await saveItems(newItems, domain, role);
// store new items in database
const curatedItemEntries = curateItemEntries(savedItems);
const storedItems = await knex('media').insert(curatedItemEntries).returning('*');
const { hash: storedItemsByHash } = groupItems(Array.isArray(storedItems) ? storedItems : []);
// accumulate existing and new items by source to be mapped onto releases
const itemsByHash = { ...existingSourceItemsByHash, ...existingHashItemsByHash, ...storedItemsByHash };
const itemsBySource = {
...existingSourceItemsBySource,
...fetchedItems.reduce((acc, item) => ({ ...acc, [item.source]: itemsByHash[item.hash] }), {}),
};
logger.info(`Stored ${fetchedItems.length} new ${domain} ${role}s`);
return itemsBySource;
}
function extractPrimaryItem(associations, targetId, role, primaryRole, primaryItemsByTargetId) {
@ -302,7 +308,7 @@ function associateTargetMedia(targetId, sources, mediaBySource, domain, role, pr
})
.filter(Boolean);
logger.info(`Associating ${associations.length} ${role}s to ${domain} ${targetId}`);
logger.silly(`Associating ${associations.length} ${role}s to ${domain} ${targetId}`);
return extractPrimaryItem(associations, targetId, role, primaryRole, primaryItemsByTargetId);
}
@ -316,6 +322,9 @@ async function associateMedia(sourcesByTargetId, mediaBySource, domain, role, pr
const associations = associationsPerTarget.map(association => association[role]).flat().filter(Boolean);
const primaryAssociations = associationsPerTarget.map(association => association[primaryRole]).filter(Boolean);
logger.info(`Associated ${associations.length} ${role}s to ${domain}s`);
logger.info(`Associated ${primaryAssociations.length} extracted ${primaryRole}s to ${domain}s`);
return Promise.all([
(associations.length > 0 && knex.raw(`${knex(`${domain}s_${role}s`).insert(associations).toString()} ON CONFLICT DO NOTHING`)),
(primaryAssociations.length > 0 && knex.raw(`${knex(`${domain}s_${primaryRole}s`).insert(primaryAssociations).toString()} ON CONFLICT DO NOTHING`)),

12
src/releases.js
View File

@ -346,12 +346,12 @@ async function storeReleaseAssets(releases) {
// ensure posters are available before fetching supplementary media
await Promise.all([
associateMedia(releasePostersById, posters, 'release', 'poster'),
associateMedia(releaseCoversById, covers, 'release', 'cover'),
(posters && associateMedia(releasePostersById, posters, 'release', 'poster')),
(covers && associateMedia(releaseCoversById, covers, 'release', 'cover')),
]);
// const photos = await storeMedia(Object.values(releasePhotosById).flat(), 'release', 'photo');
// await associateMedia(releasePhotosById, photos, 'release', 'photo');
const photos = await storeMedia(Object.values(releasePhotosById).flat(), 'release', 'photo');
if (photos) await associateMedia(releasePhotosById, photos, 'release', 'photo');
// videos take a long time, fetch last
const [trailers, teasers] = await Promise.all([
@ -360,8 +360,8 @@ async function storeReleaseAssets(releases) {
]);
await Promise.all([
associateMedia(releaseTrailersById, trailers, 'release', 'trailer'),
associateMedia(releaseTeasersById, teasers, 'release', 'teaser'),
(trailers && associateMedia(releaseTrailersById, trailers, 'release', 'trailer')),
(teasers && associateMedia(releaseTeasersById, teasers, 'release', 'teaser')),
]);
}