DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added avatars. Added PornHub and LegalPorno actor profile scrapers.

Browse Source
This commit is contained in:
ThePendulum
2019-11-20 04:53:36 +01:00
parent a13d92b84e
commit 9fcc40dd17
13 changed files with 475 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

104
src/media.js
View File

@@ -28,10 +28,19 @@ async function getThumbnail(buffer) {
.toBuffer();
}
async function createMediaDirectory(release, releaseId) {
async function createReleaseMediaDirectory(release, releaseId) {
if (release.poster || (release.photos && release.photos.length)) {
await fs.mkdir(
path.join(config.media.path, release.site.network.slug, release.site.slug, releaseId.toString()),
path.join(config.media.path, 'releases', release.site.network.slug, release.site.slug, releaseId.toString()),
{ recursive: true },
);
}
}
async function createActorMediaDirectory(profile, actor) {
if (profile.avatars && profile.avatars.length) {
await fs.mkdir(
path.join(config.media.path, 'actors', actor.slug),
{ recursive: true },
);
}
@@ -46,15 +55,16 @@ async function storePoster(release, releaseId) {
console.log(`Storing poster for (${release.site.name}, ${releaseId}) "${release.title}"`);
const res = await bhttp.get(release.poster);
const thumbnail = await getThumbnail(res.body);
if (res.statusCode === 200) {
const thumbnail = await getThumbnail(res.body);
const { pathname } = new URL(release.poster);
const mimetype = res.headers['content-type'] || mime.getType(pathname) || 'image/jpeg';
const extension = mime.getExtension(mimetype);
const filepath = path.join(release.site.network.slug, release.site.slug, releaseId.toString(), `poster.${extension}`);
const thumbpath = path.join(release.site.network.slug, release.site.slug, releaseId.toString(), `poster_thumb.${extension}`);
const filepath = path.join('releases', release.site.network.slug, release.site.slug, releaseId.toString(), `poster.${extension}`);
const thumbpath = path.join('releases', release.site.network.slug, release.site.slug, releaseId.toString(), `poster_thumb.${extension}`);
const hash = getHash(res.body);
await Promise.all([
@@ -93,13 +103,13 @@ async function storePhotos(release, releaseId) {
try {
const res = await bhttp.get(photoUrl);
const thumbnail = await getThumbnail(res.body);
if (res.statusCode === 200) {
const thumbnail = await getThumbnail(res.body);
const extension = mime.getExtension(mimetype);
const filepath = path.join(release.site.network.slug, release.site.slug, releaseId.toString(), `${index + 1}.${extension}`);
const thumbpath = path.join(release.site.network.slug, release.site.slug, releaseId.toString(), `${index + 1}_thumb.${extension}`);
const filepath = path.join('releases', release.site.network.slug, release.site.slug, releaseId.toString(), `${index + 1}.${extension}`);
const thumbpath = path.join('releases', release.site.network.slug, release.site.slug, releaseId.toString(), `${index + 1}_thumb.${extension}`);
const hash = getHash(res.body);
await Promise.all([
@@ -153,7 +163,7 @@ async function storeTrailer(release, releaseId) {
const mimetype = release.trailer.type || mime.getType(pathname);
const res = await bhttp.get(release.trailer.src);
const filepath = path.join(release.site.network.slug, release.site.slug, releaseId.toString(), `trailer${release.trailer.quality ? `_${release.trailer.quality}` : ''}.${mime.getExtension(mimetype)}`);
const filepath = path.join('releases', release.site.network.slug, release.site.slug, releaseId.toString(), `trailer${release.trailer.quality ? `_${release.trailer.quality}` : ''}.${mime.getExtension(mimetype)}`);
await Promise.all([
fs.writeFile(path.join(config.media.path, filepath), res.body),
@@ -169,8 +179,82 @@ async function storeTrailer(release, releaseId) {
]);
}
async function storeAvatars(profile, actor) {
if (!profile.avatars || profile.avatars.length === 0) {
console.warn(`No avatars available for '${profile.name}'`);
return;
}
console.log(`Storing ${profile.avatars.length} avatars for '${profile.name}'`);
const files = await Promise.map(profile.avatars, async (avatarUrl, index) => {
const { pathname } = new URL(avatarUrl);
const mimetype = mime.getType(pathname);
try {
const res = await bhttp.get(avatarUrl);
if (res.statusCode === 200) {
const thumbnail = await getThumbnail(res.body);
const extension = mime.getExtension(mimetype);
const filepath = path.join('actors', actor.slug, `${index + 1}.${extension}`);
const thumbpath = path.join('actors', actor.slug, `${index + 1}_thumb.${extension}`);
const hash = getHash(res.body);
await Promise.all([
fs.writeFile(path.join(config.media.path, filepath), res.body),
fs.writeFile(path.join(config.media.path, thumbpath), thumbnail),
]);
return {
filepath,
thumbpath,
mimetype,
hash,
source: avatarUrl,
};
}
throw new Error(`Response ${res.statusCode} not OK`);
} catch (error) {
console.warn(`Failed to store avatar ${index + 1} for '${profile.name}'`);
return null;
}
}, {
concurrency: 2,
});
const existingAvatars = await knex('media')
.whereIn('hash', files.map(file => file.hash));
const newAvatars = files.filter((file) => {
if (!file) {
return false;
}
return !existingAvatars.some(avatar => file.hash === avatar.hash);
});
await knex('media')
.insert(newAvatars.map((file, index) => ({
path: file.filepath,
thumbnail: file.thumbpath,
mime: file.mimetype,
hash: file.hash,
source: file.source,
index,
domain: 'actors',
target_id: actor.id,
role: 'avatar',
})));
}
module.exports = {
createMediaDirectory,
createActorMediaDirectory,
createReleaseMediaDirectory,
storeAvatars,
storePoster,
storePhotos,
storeTrailer,