DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Improved scraping and association behavior.

Browse Source
This commit is contained in:
ThePendulum
2019-11-19 04:36:15 +01:00
parent 7e9fd19c2a
commit 3ec7b15886
10 changed files with 166 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

40
src/media.js
View File

@@ -37,13 +37,13 @@ async function createMediaDirectory(release, releaseId) {
}
}
async function storePoster(release, releaseEntry) {
async function storePoster(release, releaseId) {
if (!release.poster) {
console.warn(`No poster available for (${release.site.name}, ${releaseEntry.id}}) "${release.title}"`);
console.warn(`No poster available for (${release.site.name}, ${releaseId}}) "${release.title}"`);
return;
}
console.log(`Storing poster for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
console.log(`Storing poster for (${release.site.name}, ${releaseId}) "${release.title}"`);
const res = await bhttp.get(release.poster);
const thumbnail = await getThumbnail(res.body);
@@ -53,8 +53,8 @@ async function storePoster(release, releaseEntry) {
const mimetype = res.headers['content-type'] || mime.getType(pathname) || 'image/jpeg';
const extension = mime.getExtension(mimetype);
const filepath = path.join(release.site.network.slug, release.site.slug, releaseEntry.id.toString(), `poster.${extension}`);
const thumbpath = path.join(release.site.network.slug, release.site.slug, releaseEntry.id.toString(), `poster_thumb.${extension}`);
const filepath = path.join(release.site.network.slug, release.site.slug, releaseId.toString(), `poster.${extension}`);
const thumbpath = path.join(release.site.network.slug, release.site.slug, releaseId.toString(), `poster_thumb.${extension}`);
const hash = getHash(res.body);
await Promise.all([
@@ -69,23 +69,23 @@ async function storePoster(release, releaseEntry) {
hash,
source: release.poster,
domain: 'releases',
target_id: releaseEntry.id,
target_id: releaseId,
role: 'poster',
});
return;
}
console.warn(`Failed to store poster for (${release.site.name}, ${releaseEntry.id}) "${release.title}": ${res.statusCode}`);
console.warn(`Failed to store poster for (${release.site.name}, ${releaseId}) "${release.title}": ${res.statusCode}`);
}
async function storePhotos(release, releaseEntry) {
if (release.photos.length === 0) {
console.warn(`No photos available for (${release.site.name}, ${releaseEntry.id}}) "${release.title}"`);
async function storePhotos(release, releaseId) {
if (!release.photos || release.photos.length === 0) {
console.warn(`No photos available for (${release.site.name}, ${releaseId}}) "${release.title}"`);
return;
}
console.log(`Storing ${release.photos.length} photos for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
console.log(`Storing ${release.photos.length} photos for (${release.site.name}, ${releaseId}) "${release.title}"`);
const files = await Promise.map(release.photos, async (photoUrl, index) => {
const { pathname } = new URL(photoUrl);
@@ -98,8 +98,8 @@ async function storePhotos(release, releaseEntry) {
if (res.statusCode === 200) {
const extension = mime.getExtension(mimetype);
const filepath = path.join(release.site.network.slug, release.site.slug, releaseEntry.id.toString(), `${index + 1}.${extension}`);
const thumbpath = path.join(release.site.network.slug, release.site.slug, releaseEntry.id.toString(), `${index + 1}_thumb.${extension}`);
const filepath = path.join(release.site.network.slug, release.site.slug, releaseId.toString(), `${index + 1}.${extension}`);
const thumbpath = path.join(release.site.network.slug, release.site.slug, releaseId.toString(), `${index + 1}_thumb.${extension}`);
const hash = getHash(res.body);
await Promise.all([
@@ -118,7 +118,7 @@ async function storePhotos(release, releaseEntry) {
throw new Error(`Response ${res.statusCode} not OK`);
} catch (error) {
console.warn(`Failed to store photo ${index + 1} for "${release.title}" (${photoUrl}, ${release.url}, ${release.site.name}, ${releaseEntry.id}): ${error}`);
console.warn(`Failed to store photo ${index + 1} for "${release.title}" (${photoUrl}, ${release.url}, ${release.site.name}, ${releaseId}): ${error}`);
return null;
}
@@ -136,24 +136,24 @@ async function storePhotos(release, releaseEntry) {
source: file.source,
index,
domain: 'releases',
target_id: releaseEntry.id,
target_id: releaseId,
role: 'photo',
})));
}
async function storeTrailer(release, releaseEntry) {
async function storeTrailer(release, releaseId) {
if (!release.trailer || !release.trailer.src) {
console.warn(`No trailer available for (${release.site.name}, ${releaseEntry.id}}) "${release.title}"`);
console.warn(`No trailer available for (${release.site.name}, ${releaseId}}) "${release.title}"`);
return;
}
console.log(`Storing trailer for (${release.site.name}, ${releaseEntry.id}) "${release.title}"`);
console.log(`Storing trailer for (${release.site.name}, ${releaseId}) "${release.title}"`);
const { pathname } = new URL(release.trailer.src);
const mimetype = release.trailer.type || mime.getType(pathname);
const res = await bhttp.get(release.trailer.src);
const filepath = path.join(release.site.network.slug, release.site.slug, releaseEntry.id.toString(), `trailer${release.trailer.quality ? `_${release.trailer.quality}` : ''}.${mime.getExtension(mimetype)}`);
const filepath = path.join(release.site.network.slug, release.site.slug, releaseId.toString(), `trailer${release.trailer.quality ? `_${release.trailer.quality}` : ''}.${mime.getExtension(mimetype)}`);
await Promise.all([
fs.writeFile(path.join(config.media.path, filepath), res.body),
@@ -162,7 +162,7 @@ async function storeTrailer(release, releaseEntry) {
mime: mimetype,
source: release.trailer.src,
domain: 'releases',
target_id: releaseEntry.id,
target_id: releaseId,
role: 'trailer',
quality: release.trailer.quality || null,
}),