Compare commits
No commits in common. "f684923a8adf0baa15165258a7d3d7260a2843e5" and "2ac879d27600fe56a6e4c345096e2517d8fa75d7" have entirely different histories.
f684923a8a
...
2ac879d276
|
|
@ -136,7 +136,7 @@ export default {
|
|||
|
||||
.tiles {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(10rem, 1fr));
|
||||
grid-template-columns: repeat(auto-fit, 10rem);
|
||||
grid-gap: 0 .5rem;
|
||||
padding: 1rem;
|
||||
flex-grow: 1;
|
||||
|
|
@ -209,7 +209,7 @@ export default {
|
|||
|
||||
@media(max-width: $breakpoint) {
|
||||
.tiles {
|
||||
grid-template-columns: repeat(auto-fill, minmax(8rem, 1fr));
|
||||
grid-template-columns: repeat(auto-fit, minmax(8rem, 1fr));
|
||||
}
|
||||
|
||||
.genders {
|
||||
|
|
|
|||
|
|
@ -400,14 +400,13 @@ export default {
|
|||
}
|
||||
|
||||
.actors {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fill, minmax(10rem, 1fr));
|
||||
grid-gap: 1rem;
|
||||
display: flex;
|
||||
flex-grow: 1;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.actor {
|
||||
width: 10rem;
|
||||
margin: 0 1rem .5rem 0;
|
||||
}
|
||||
|
||||
|
|
@ -478,9 +477,5 @@ export default {
|
|||
width: 15rem;
|
||||
max-width: 100%;
|
||||
}
|
||||
|
||||
.actors {
|
||||
grid-template-columns: repeat(auto-fill, minmax(8rem, 1fr));
|
||||
}
|
||||
}
|
||||
</style>
|
||||
|
|
|
|||
|
|
@ -22,10 +22,8 @@
|
|||
<span
|
||||
v-else
|
||||
v-tooltip.top="actor.name"
|
||||
class="handle"
|
||||
>
|
||||
<span class="name">{{ actor.name }}</span>
|
||||
</span>
|
||||
class="handle name"
|
||||
>{{ actor.name }}</span>
|
||||
|
||||
<div class="avatar-container">
|
||||
<img
|
||||
|
|
@ -105,28 +103,12 @@ export default {
|
|||
.actor {
|
||||
width: 100%;
|
||||
display: inline-block;
|
||||
position: relative;
|
||||
margin: 0 .5rem .5rem 0;
|
||||
box-shadow: 0 0 3px var(--darken-weak);
|
||||
background: var(--profile);
|
||||
|
||||
&::before {
|
||||
content: '';
|
||||
display: inline-block;
|
||||
width: 1px;
|
||||
height: 0;
|
||||
padding-bottom: 150%;
|
||||
}
|
||||
}
|
||||
|
||||
.link {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
position: absolute;
|
||||
top: 0;
|
||||
left: 0;
|
||||
color: var(--text-light);
|
||||
text-decoration: none;
|
||||
|
||||
|
|
@ -162,18 +144,15 @@ export default {
|
|||
}
|
||||
|
||||
.avatar-container {
|
||||
display: flex;
|
||||
flex-grow: 1;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.avatar {
|
||||
color: var(--darken-weak);
|
||||
background: var(--darken-hint);
|
||||
height: 13rem;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
object-fit: cover;
|
||||
|
|
|
|||
59
src/media.js
59
src/media.js
|
|
@ -334,7 +334,7 @@ async function fetchSource(source, baseMedia) {
|
|||
}
|
||||
|
||||
const { pathname } = new URL(source.src);
|
||||
const mimetype = res.headers['content-type'] || mime.getType(pathname);
|
||||
const mimetype = mime.getType(pathname);
|
||||
const extension = mime.getExtension(mimetype);
|
||||
const type = mimetype?.split('/')[0] || 'image';
|
||||
|
||||
|
|
@ -342,9 +342,7 @@ async function fetchSource(source, baseMedia) {
|
|||
hasher.setEncoding('hex');
|
||||
|
||||
const hashStream = new PassThrough();
|
||||
const metaStream = type === 'image'
|
||||
? sharp()
|
||||
: new PassThrough();
|
||||
const metaStream = type === 'image' ? sharp() : new PassThrough();
|
||||
|
||||
const tempFilePath = path.join(config.media.path, 'temp', `${baseMedia.id}.${extension}`);
|
||||
const tempThumbPath = path.join(config.media.path, 'temp', `${baseMedia.id}_thumb.${extension}`);
|
||||
|
|
@ -355,8 +353,6 @@ async function fetchSource(source, baseMedia) {
|
|||
hashStream.on('data', chunk => hasher.write(chunk));
|
||||
|
||||
if (type === 'image') {
|
||||
// generate thumbnail
|
||||
/*
|
||||
metaStream
|
||||
.clone()
|
||||
.resize({
|
||||
|
|
@ -366,21 +362,34 @@ async function fetchSource(source, baseMedia) {
|
|||
.jpeg({ quality: config.media.thumbnailQuality })
|
||||
.pipe(tempThumbTarget)
|
||||
.on('error', error => logger.error(error));
|
||||
*/
|
||||
}
|
||||
|
||||
// pipeline destroys streams, so attach info event first
|
||||
// pipeline destroys streams
|
||||
const infoPromise = type === 'image' ? once(metaStream, 'info') : Promise.resolve([{}]);
|
||||
const metaPromise = type === 'image' ? metaStream.stats() : Promise.resolve();
|
||||
|
||||
await pipeline(
|
||||
res.originalRes,
|
||||
// metaStream,
|
||||
metaStream,
|
||||
hashStream,
|
||||
tempFileTarget,
|
||||
);
|
||||
|
||||
const [stats, info] = await Promise.all([metaPromise, infoPromise]);
|
||||
/*
|
||||
res.originalRes
|
||||
.pipe(metaStream)
|
||||
.pipe(hashStream)
|
||||
.pipe(tempFileTarget);
|
||||
*/
|
||||
|
||||
logger.silly(`Temporarily saved media from ${source.src}`);
|
||||
|
||||
const [stats, info] = await Promise.all([
|
||||
metaPromise,
|
||||
infoPromise,
|
||||
]);
|
||||
|
||||
logger.silly(`Ended pipeline for ${source.src}`);
|
||||
|
||||
hasher.end();
|
||||
|
||||
|
|
@ -389,7 +398,7 @@ async function fetchSource(source, baseMedia) {
|
|||
|
||||
peakMemoryUsage = Math.max(getMemoryUsage(), peakMemoryUsage);
|
||||
|
||||
logger.silly(`Fetched media from ${source.src}, memory usage ${peakMemoryUsage.toFixed(2)} MB`);
|
||||
logger.silly(`Retrieved metadata from ${source.src}`);
|
||||
|
||||
return {
|
||||
...source,
|
||||
|
|
@ -413,21 +422,14 @@ async function fetchSource(source, baseMedia) {
|
|||
|
||||
if (attempts < 3) {
|
||||
await Promise.delay(1000);
|
||||
|
||||
return Promise.race([
|
||||
attempt(attempts + 1),
|
||||
Promise.delay(120 * 1000).then(() => { throw new Error(`Media fetch attempt ${attempts}/3 timed out, aborting ${source.src}`); }),
|
||||
]);
|
||||
return attempt(attempts + 1);
|
||||
}
|
||||
|
||||
throw new Error(`Failed to fetch ${source.src}: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
return Promise.race([
|
||||
attempt(1),
|
||||
Promise.delay(120 * 1000).then(() => { throw new Error(`Media fetch timed out, aborting ${source.src}`); }),
|
||||
]);
|
||||
return attempt(1);
|
||||
}
|
||||
|
||||
async function trySource(baseSource, existingMedias, baseMedia) {
|
||||
|
|
@ -435,14 +437,14 @@ async function trySource(baseSource, existingMedias, baseMedia) {
|
|||
const extractedSource = await extractSource(baseSource, existingMedias);
|
||||
const existingSourceMedia = existingMedias.existingSourceMediaByUrl[extractedSource.src];
|
||||
|
||||
if (!argv.force && extractedSource.entry) {
|
||||
if (extractedSource.entry) {
|
||||
logger.silly(`Media page URL already in database, not extracting ${baseSource.url}`);
|
||||
|
||||
// media entry found during extraction, don't fetch
|
||||
return extractedSource;
|
||||
}
|
||||
|
||||
if (!argv.force && existingSourceMedia) {
|
||||
if (existingSourceMedia) {
|
||||
logger.silly(`Media source URL already in database, skipping ${baseSource.src}`);
|
||||
|
||||
// media entry found by source URL, don't fetch
|
||||
|
|
@ -459,13 +461,7 @@ async function fetchMedia(baseMedia, existingMedias) {
|
|||
try {
|
||||
const source = await baseMedia.sources.reduce(
|
||||
// try each source until success
|
||||
(result, baseSource, baseSourceIndex) => result.catch(async (error) => {
|
||||
if (error.message) {
|
||||
logger.warn(error.message);
|
||||
}
|
||||
|
||||
return trySource(baseSource, existingMedias, baseMedia, baseSourceIndex);
|
||||
}),
|
||||
(result, baseSource, baseSourceIndex) => result.catch(async () => trySource(baseSource, existingMedias, baseMedia, baseSourceIndex)),
|
||||
Promise.reject(new Error()),
|
||||
);
|
||||
|
||||
|
|
@ -533,6 +529,7 @@ async function storeMedias(baseMedias) {
|
|||
const savedMedias = await Promise.map(
|
||||
baseMedias,
|
||||
async baseMedia => fetchMedia(baseMedia, { existingSourceMediaByUrl, existingExtractMediaByUrl }),
|
||||
{ concurrency: 10 },
|
||||
);
|
||||
|
||||
const [uniqueHashMedias, existingHashMedias] = await findHashDuplicates(savedMedias);
|
||||
|
|
@ -602,9 +599,7 @@ async function associateReleaseMedia(releases) {
|
|||
}, [])
|
||||
.filter(Boolean);
|
||||
|
||||
if (associations.length > 0) {
|
||||
await knex.raw(`${knex(`releases_${role}`).insert(associations)} ON CONFLICT DO NOTHING`);
|
||||
}
|
||||
await knex.raw(`${knex(`releases_${role}`).insert(associations)} ON CONFLICT DO NOTHING`);
|
||||
}, Promise.resolve());
|
||||
|
||||
logger.debug(`Peak media fetching memory usage: ${peakMemoryUsage.toFixed(2)} MB`);
|
||||
|
|
|
|||
|
|
@ -42,32 +42,6 @@ function getAvatarFallbacks(avatar) {
|
|||
.flat();
|
||||
}
|
||||
|
||||
async function getTrailer(scene, site, url) {
|
||||
const qualities = [360, 480, 720, 1080, 2160];
|
||||
|
||||
const tokenRes = await post(`${site.url}/api/__record_tknreq`, {
|
||||
file: scene.previewVideoUrl1080P,
|
||||
sizes: qualities.join('+'),
|
||||
type: 'trailer',
|
||||
}, { referer: url });
|
||||
|
||||
if (!tokenRes.ok) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const trailerUrl = `${site.url}/api${tokenRes.body.data.url}`;
|
||||
const trailersRes = await post(trailerUrl, null, { referer: url });
|
||||
|
||||
if (trailersRes.ok) {
|
||||
return qualities.map(quality => (trailersRes.body[quality] ? {
|
||||
src: trailersRes.body[quality].token,
|
||||
quality,
|
||||
} : null)).filter(Boolean);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function scrapeAll(scenes, site, origin) {
|
||||
return scenes.map((scene) => {
|
||||
const release = {};
|
||||
|
|
@ -116,6 +90,32 @@ function scrapeUpcoming(scene, site) {
|
|||
return [release];
|
||||
}
|
||||
|
||||
async function getTrailer(scene, site, url) {
|
||||
const qualities = [360, 480, 720, 1080, 2160];
|
||||
|
||||
const tokenRes = await post(`${site.url}/api/__record_tknreq`, {
|
||||
file: scene.previewVideoUrl1080P,
|
||||
sizes: qualities.join('+'),
|
||||
type: 'trailer',
|
||||
}, { referer: url });
|
||||
|
||||
if (!tokenRes.ok) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const trailerUrl = `${site.url}/api${tokenRes.body.data.url}`;
|
||||
const trailersRes = await post(trailerUrl, null, { referer: url });
|
||||
|
||||
if (trailersRes.ok) {
|
||||
return qualities.map(quality => (trailersRes.body[quality] ? {
|
||||
src: trailersRes.body[quality].token,
|
||||
quality,
|
||||
} : null)).filter(Boolean);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
async function scrapeScene(data, url, site, baseRelease) {
|
||||
const scene = data.video;
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue