diff --git a/assets/components/actors/actors.vue b/assets/components/actors/actors.vue
index f38894b21..ecbed75c5 100644
--- a/assets/components/actors/actors.vue
+++ b/assets/components/actors/actors.vue
@@ -116,7 +116,7 @@ export default {
.tiles {
display: grid;
- grid-template-columns: repeat(auto-fit, minmax(10rem, .5fr));
+ grid-template-columns: repeat(auto-fit, 10rem);
grid-gap: 0 .5rem;
padding: 1rem;
flex-grow: 1;
@@ -127,10 +127,11 @@ export default {
justify-content: center;
align-items: center;
padding: 0 1rem;
- margin: 1rem 0 0 0;
+ margin: 1rem 0;
}
.genders {
+ flex-shrink: 0;
padding: 0 .5rem 0 0;
border-right: solid 1px $shadow-hint;
margin: 0 1rem 0 0;
@@ -186,7 +187,7 @@ export default {
}
@media(max-width: $breakpoint) {
- .actors {
+ .tiles {
grid-template-columns: repeat(auto-fit, minmax(8rem, 1fr));
}
}
diff --git a/assets/components/tags/tags.vue b/assets/components/tags/tags.vue
index 0f4f1b02f..e255a080c 100644
--- a/assets/components/tags/tags.vue
+++ b/assets/components/tags/tags.vue
@@ -147,7 +147,19 @@ export default {
.tiles {
display: grid;
- grid-template-columns: repeat(auto-fit, minmax(20rem, 1fr));
+ grid-template-columns: repeat(auto-fit, minmax(20rem, .25fr));
grid-gap: .5rem;
}
+
+@media(max-width: $breakpoint3) {
+ .tiles {
+ grid-template-columns: repeat(auto-fit, minmax(20rem, .5fr));
+ }
+}
+
+@media(max-width: $breakpoint) {
+ .tiles {
+ grid-template-columns: repeat(auto-fit, minmax(20rem, 1fr));
+ }
+}
diff --git a/assets/components/tile/release.vue b/assets/components/tile/release.vue
index 205d8849f..0468284cc 100644
--- a/assets/components/tile/release.vue
+++ b/assets/components/tile/release.vue
@@ -48,8 +48,10 @@
>{{ `(${formatDate(release.dateAdded, 'MMM D, YYYY')})` }}
-
![]()
No thumbnail available
-
+
-
{{ release.title }}
-
+
diff --git a/config/default.js b/config/default.js
index 65608468d..449cc11f4 100644
--- a/config/default.js
+++ b/config/default.js
@@ -98,7 +98,7 @@ module.exports = {
path: './media',
thumbnailSize: 320, // width for 16:9 will be exactly 576px
thumbnailQuality: 100,
- trailerQuality: [480, 540],
+ videoQuality: [480, 360, 320, 540, 720, 1080, 2160],
limit: 25, // max number of photos per release
},
titleSlugLength: 50,
diff --git a/public/img/tags/gaping/1.jpeg b/public/img/tags/gaping/1.jpeg
new file mode 100644
index 000000000..38c3828e4
Binary files /dev/null and b/public/img/tags/gaping/1.jpeg differ
diff --git a/public/img/tags/gaping/1_thumb.jpeg b/public/img/tags/gaping/1_thumb.jpeg
new file mode 100644
index 000000000..f417d078c
Binary files /dev/null and b/public/img/tags/gaping/1_thumb.jpeg differ
diff --git a/seeds/04_media.js b/seeds/04_media.js
index b038ae42d..c190d58cc 100644
--- a/seeds/04_media.js
+++ b/seeds/04_media.js
@@ -1,39 +1,39 @@
const upsert = require('../src/utils/upsert');
-const tagPosters = Object.entries({
- 'anal-creampie': [0, 'Gina Valentina and Jane Wilde in "A Very Special Anniversary" for Tushy'],
- 'ass-to-mouth': ['poster', 'Alysa Gap and Logan in "Anal Buffet 4" for Evil Angel'],
- 'da-tp': [0, 'Natasha Teen in LegalPorno SZ2164'],
- 'double-anal': [5, 'Riley Reid in "The Gangbang of Riley Reid" for Jules Jordan'],
- 'double-penetration': ['poster', 'Mia Malkova in "DP!" for HardX'],
- 'double-vaginal': ['poster', 'Riley Reid in "Pizza That Ass" for Reid My Lips'],
- 'dv-tp': ['poster', 'Juelz Ventura in "Gangbanged 5" for Elegant Angel'],
- 'oral-creampie': [1, 'Keisha Grey in Brazzers House'],
- 'triple-anal': ['poster', 'Kristy Black in SZ1986 for LegalPorno'],
- airtight: [1, 'Jynx Maze in "Pump My Ass Full of Cum 3" for Jules Jordan'],
- anal: ['poster', 'Jynx Maze in "Anal Buffet 6" for Evil Angel'],
- asian: ['poster', 'Vina Sky in "Young and Glamorous 10" for Jules Jordan'],
- blowjob: [0, 'Adriana Chechik in "The Dinner Party" for Real Wife Stories (Brazzers)'],
- blowbang: ['poster'],
- bukkake: ['poster'],
- caucasian: ['poster'],
- creampie: ['poster'],
- ebony: [1, 'Sarah Banks for Brazzers'],
- facial: ['poster'],
- facefucking: ['1', 'Carrie for Young Throats'],
- gangbang: ['poster', 'Kristen Scott in "Interracial Gangbang!" for Jules Jordan'],
- gaping: [0, 'McKenzee Miles in "Anal Buffet 4" for Evil Angel'],
- interracial: ['poster'],
- latina: ['poster'],
- mff: ['poster'],
- mfm: ['poster'],
- orgy: ['poster'],
- schoolgirl: [1, 'Eliza Ibarra for Brazzers'],
- swallowing: ['poster'],
- tattoo: ['poster', 'Kali Roses in "Goes All In For Anal" for Hussie Pass'],
- trainbang: ['poster', 'Kali Roses in "Passing Me Around" for Blacked'],
-})
- .map(([slug, [filename, comment]], index) => ({
+const tagPosters = [
+ ['anal-creampie', 0, 'Gina Valentina and Jane Wilde in "A Very Special Anniversary" for Tushy'],
+ ['ass-to-mouth', 'poster', 'Alysa Gap and Logan in "Anal Buffet 4" for Evil Angel'],
+ ['da-tp', 0, 'Natasha Teen in LegalPorno SZ2164'],
+ ['double-anal', 5, 'Riley Reid in "The Gangbang of Riley Reid" for Jules Jordan'],
+ ['double-penetration', 'poster', 'Mia Malkova in "DP!" for HardX'],
+ ['double-vaginal', 'poster', 'Riley Reid in "Pizza That Ass" for Reid My Lips'],
+ ['dv-tp', 'poster', 'Juelz Ventura in "Gangbanged 5" for Elegant Angel'],
+ ['oral-creampie', 1, 'Keisha Grey in Brazzers House'],
+ ['triple-anal', 'poster', 'Kristy Black in SZ1986 for LegalPorno'],
+ ['airtight', 1, 'Jynx Maze in "Pump My Ass Full of Cum 3" for Jules Jordan'],
+ ['anal', 'poster', 'Jynx Maze in "Anal Buffet 6" for Evil Angel'],
+ ['asian', 'poster', 'Vina Sky in "Young and Glamorous 10" for Jules Jordan'],
+ ['blowjob', 0, 'Adriana Chechik in "The Dinner Party" for Real Wife Stories (Brazzers)'],
+ ['blowbang', 'poster'],
+ ['bukkake', 'poster'],
+ ['caucasian', 'poster'],
+ ['creampie', 'poster'],
+ ['ebony', 1, 'Sarah Banks for Brazzers'],
+ ['facial', 'poster'],
+ ['facefucking', '1', 'Carrie for Young Throats'],
+ ['gangbang', 'poster', 'Kristen Scott in "Interracial Gangbang!" for Jules Jordan'],
+ ['gaping', 1, 'Vina Sky in "Vina Sky Does Anal" for HardX'],
+ ['interracial', 'poster'],
+ ['latina', 'poster'],
+ ['mff', 'poster'],
+ ['mfm', 'poster'],
+ ['orgy', 'poster'],
+ ['schoolgirl', 1, 'Eliza Ibarra for Brazzers'],
+ ['swallowing', 'poster'],
+ ['tattoo', 'poster', 'Kali Roses in "Goes All In For Anal" for Hussie Pass'],
+ ['trainbang', 'poster', 'Kali Roses in "Passing Me Around" for Blacked'],
+]
+ .map(([slug, filename, comment], index) => ({
tagSlug: slug,
path: `tags/${slug}/${filename}.jpeg`,
thumbnail: `tags/${slug}/${filename}_thumb.jpeg`,
@@ -63,6 +63,7 @@ const tagPhotos = [
['gangbang', 1, 'Ginger Lynn in "Gangbang Mystique", a photoset shot by Suze Randall for Puritan No. 10, 1984. This photo pushed the boundaries of pornography at the time, as depicting a woman \'fully occupied\' was unheard of.'],
['gangbang', 2, 'Riley Reid\'s double anal in "The Gangbang of Riley Reid" for Jules Jordan'],
['gaping', 'poster', 'Paulina in "Anal Buffet 4" for Evil Angel'],
+ ['gaping', 0, 'McKenzee Miles in "Anal Buffet 4" for Evil Angel'],
['trainbang', 0, 'Nicole Black in GIO971 for LegalPorno'],
['triple-anal', 1, 'Natasha Teen in SZ2098 for LegalPorno'],
['triple-anal', 2, 'Kira Thorn in GIO1018 for LegalPorno'],
diff --git a/src/actors.js b/src/actors.js
index 2aa0e06b5..9087d12b7 100644
--- a/src/actors.js
+++ b/src/actors.js
@@ -12,7 +12,8 @@ const scrapers = require('./scrapers/scrapers');
const whereOr = require('./utils/where-or');
const resolvePlace = require('./utils/resolve-place');
const slugify = require('./utils/slugify');
-const { createMediaDirectory, storePhotos } = require('./media_legacy');
+// const { createMediaDirectory, storePhotos } = require('./media_legacy');
+const { storeMedia, associateMedia } = require('./media');
async function curateActor(actor) {
const [aliases, avatar, photos, social] = await Promise.all([
@@ -250,6 +251,17 @@ async function storeSocialLinks(urls, actorId) {
await knex('actors_social').insert(curatedSocialEntries);
}
+async function storeAvatars(avatars, actorId) {
+ if (!avatars || avatars.length === 0) {
+ return [];
+ }
+
+ const avatarsBySource = await storeMedia(avatars, 'actor', 'avatar');
+ await associateMedia({ [actorId]: avatars }, avatarsBySource, 'actor', 'photo', 'avatar');
+
+ return avatarsBySource;
+}
+
async function storeActor(actor, scraped = false, scrapeSuccess = false) {
const curatedActor = curateActorEntry(actor, scraped, scrapeSuccess);
@@ -260,15 +272,7 @@ async function storeActor(actor, scraped = false, scrapeSuccess = false) {
await storeSocialLinks(actor.social, actorEntry.id);
if (actor.avatars) {
- await createMediaDirectory('actors', `${actorEntry.slug}/`);
- await storePhotos(actor.avatars, {
- domain: 'actor',
- role: 'photo',
- primaryRole: 'avatar',
- targetId: actorEntry.id,
- subpath: `${actorEntry.slug}/`,
- naming: 'timestamp',
- }, actorEntry.name);
+ await storeAvatars(actor.avatars, actorEntry.id);
}
logger.info(`Added new entry for actor '${actor.name}'`);
@@ -421,19 +425,9 @@ async function scrapeActors(actorNames) {
if (argv.save) {
if (actorEntry && profile) {
- await createMediaDirectory('actors', `${actorEntry.slug}/`);
-
await Promise.all([
updateActor(profile, true, true),
- // storeAvatars(profile, actorEntry),
- storePhotos(profile.avatars, {
- domain: 'actor',
- role: 'photo',
- primaryRole: 'avatar',
- targetId: actorEntry.id,
- subpath: `${actorEntry.slug}/`,
- naming: 'timestamp',
- }, actorEntry.name),
+ storeAvatars(profile.avatars, actorEntry.id),
]);
return profile;
diff --git a/src/media.js b/src/media.js
index ed52cabe4..30cd6aa86 100644
--- a/src/media.js
+++ b/src/media.js
@@ -4,11 +4,14 @@ const config = require('config');
const Promise = require('bluebird');
const bhttp = require('bhttp');
const mime = require('mime');
+const fs = require('fs-extra');
const sharp = require('sharp');
+const path = require('path');
const blake2 = require('blake2');
-const logger = require('./logger');
+const logger = require('./logger')(__filename);
const knex = require('./knex');
+const { ex } = require('./utils/q');
function getHash(buffer) {
const hash = blake2.createHash('blake2b', { digestLength: 24 });
@@ -17,6 +20,26 @@ function getHash(buffer) {
return hash.digest('hex');
}
+async function createThumbnail(buffer) {
+ try {
+ const thumbnail = sharp(buffer)
+ .resize({
+ height: config.media.thumbnailSize,
+ withoutEnlargement: true,
+ })
+ .jpeg({
+ quality: config.media.thumbnailQuality,
+ })
+ .toBuffer();
+
+ return thumbnail;
+ } catch (error) {
+ logger.error(`Failed to create thumbnail: ${error.message}`);
+ }
+
+ return null;
+}
+
function pluckItems(items, specifiedLimit) {
const limit = specifiedLimit || config.media.limit;
@@ -30,6 +53,13 @@ function pluckItems(items, specifiedLimit) {
return Array.from(new Set(plucked)).map(itemIndex => items[itemIndex - 1]); // remove duplicates, may happen when photo total and photo limit are close
}
+function pickQuality(items) {
+ const itemsByQuality = items.reduce((acc, item) => ({ ...acc, [item.quality]: item }), {});
+ const item = config.media.videoQuality.reduce((acc, quality) => acc || itemsByQuality[quality], null);
+
+ return item || items[0];
+}
+
async function getEntropy(buffer) {
try {
const { entropy } = await sharp(buffer).stats();
@@ -42,33 +72,58 @@ async function getEntropy(buffer) {
}
}
-async function fetchItem(source, index, existingItemsBySource, attempt = 1) {
+async function extractItem(source) {
+ const res = await bhttp.get(source.src);
+
+ if (res.statusCode === 200) {
+ const { q } = ex(res.body.toString());
+
+ return source.extract(q);
+ }
+
+ return null;
+}
+
+async function fetchItem(source, index, existingItemsBySource, domain, role, attempt = 1, originalSource = null) {
try {
if (Array.isArray(source)) {
+ if (source.every(sourceX => !!sourceX.quality)) {
+ // various video qualities provided
+ const selectedSource = pickQuality(source);
+ return fetchItem(selectedSource, index, existingItemsBySource, domain, role, attempt, originalSource);
+ }
+
// fallbacks provided
- return source.reduce((outcome, sourceX) => outcome.catch(async () => {
- const item = await fetchItem(sourceX, index, existingItemsBySource);
-
- if (item) {
- return item;
- }
-
- throw new Error(`Item not available: ${source}`);
- }), Promise.reject(new Error()));
+ return source.reduce(
+ (outcome, sourceX) => outcome.catch(async () => fetchItem(sourceX, index, existingItemsBySource, domain, role, attempt, originalSource)),
+ Promise.reject(new Error()),
+ );
}
+ if (source.src && source.extract) {
+ // source links to page containing a (presumably) tokenized photo
+ const itemSource = await extractItem(source);
+
+ return fetchItem(itemSource, index, existingItemsBySource, domain, role, attempt, source);
+ }
+
+
if (existingItemsBySource[source]) {
- return existingItemsBySource[source];
+ return null;
}
- const res = await bhttp.get(source);
+ logger.verbose(`Fetching media item from ${source.src || source}`);
+
+ const res = await bhttp.get(source.src || source);
if (res.statusCode === 200) {
- const { pathname } = new URL(source);
+ const { pathname } = new URL(source.src || source);
const mimetype = mime.getType(pathname);
const extension = mime.getExtension(mimetype);
const hash = getHash(res.body);
- const entropy = await getEntropy(res.body);
+ const entropy = /image/.test(mimetype) ? await getEntropy(res.body) : null;
+
+ logger.verbose(`Fetched media item from ${source.src || source}`);
return {
file: res.body,
@@ -76,40 +131,193 @@ async function fetchItem(source, index, existingItemsBySource, attempt = 1) {
extension,
hash,
entropy,
- source,
+ quality: source.quality || null,
+ source: originalSource?.src || originalSource || source.src || source,
};
}
throw new Error(`Response ${res.statusCode} not OK`);
} catch (error) {
- if (attempt <= 3) {
- return fetchItem(source, index, existingItemsBySource, attempt + 1);
+ logger.warn(`Failed attempt ${attempt}/3 to fetch ${domain} ${role} ${index + 1} (${source.src || source}): ${error}`);
+
+ if (attempt < 3) {
+ await Promise.delay(5000);
+ return fetchItem(source, index, existingItemsBySource, domain, role, attempt + 1);
}
- throw new Error(`Failed to fetch media from ${source}: ${error}`);
+ return null;
}
}
-async function fetchItems(itemSources, existingItemsBySource) {
- return Promise.map(itemSources, async (source, index) => fetchItem(source, index, existingItemsBySource));
+async function fetchItems(itemSources, existingItemsBySource, domain, role) {
+ return Promise.map(itemSources, async (source, index) => fetchItem(source, index, existingItemsBySource, domain, role), {
+ concurrency: 10,
+ }).filter(Boolean);
}
-async function storeReleaseMedia(releases, {
- type = 'poster',
-} = {}) {
- const pluckedSources = releases.map(release => pluckItems(release[type]));
- const existingSourceItems = await knex('media').whereIn('source', pluckedSources.flat());
- const existingItemsBySource = existingSourceItems.reduce((acc, item) => ({ ...acc, [item.source]: item }), {});
+async function saveItems(items, domain, role) {
+ return Promise.map(items, async (item) => {
+ try {
+ const dir = item.hash.slice(0, 2);
+ const subdir = item.hash.slice(2, 4);
+ const filename = item.quality
+ ? `${item.hash.slice(4)}_${item.quality}.${item.extension}`
+ : `${item.hash.slice(4)}.${item.extension}`;
- const fetchedItems = await fetchItems(pluckedSources, existingItemsBySource);
- const existingHashItems = await knex('media').whereIn('hash', fetchedItems.map(item => item.hash));
- const existingItemsByHash = existingHashItems.reduce((acc, item) => ({ ...acc, [item.hash]: item }), {});
+ const filedir = path.join(`${domain}s`, `${role}s`, dir, subdir);
+ const filepath = path.join(filedir, filename);
- const newItems = fetchedItems.filter(item => !existingItemsByHash[item.hash]);
+ await fs.mkdir(path.join(config.media.path, filedir), { recursive: true });
+ await fs.writeFile(path.join(config.media.path, filepath), item.file);
- console.log(fetchedItems, existingHashItems, existingItemsByHash, newItems);
+ if (/image/.test(item.mimetype)) {
+ const thumbnail = await createThumbnail(item.file);
+
+ const thumbdir = path.join(`${domain}s`, `${role}s`, 'thumbs', dir, subdir);
+ const thumbpath = path.join(thumbdir, filename);
+
+ await fs.mkdir(path.join(config.media.path, thumbdir), { recursive: true });
+ await fs.writeFile(path.join(config.media.path, thumbpath), thumbnail);
+
+ logger.verbose(`Saved ${domain} ${role} with thumbnail to ${filepath}`);
+
+ return {
+ ...item,
+ thumbnail,
+ filepath,
+ thumbpath,
+ };
+ }
+
+ logger.verbose(`Saved ${domain} ${role} to ${filepath}`);
+
+ return {
+ ...item,
+ filepath,
+ };
+ } catch (error) {
+ logger.error(`Failed to store ${domain} ${role} from ${item.source}: ${error.message}`);
+ return null;
+ }
+ });
+}
+
+function curateItemEntries(items) {
+ return items.filter(Boolean).map((item, index) => ({
+ path: item.filepath,
+ thumbnail: item.thumbpath,
+ mime: item.mimetype,
+ hash: item.hash,
+ source: item.source,
+ entropy: item.entropy,
+ index,
+ }));
+}
+
+function groupItems(items) {
+ return items.reduce((acc, item) => ({
+ source: { ...acc.source, [item.source]: item },
+ hash: { ...acc.hash, [item.hash]: item },
+ }), {
+ source: {},
+ hash: {},
+ });
+}
+
+async function storeMedia(sources, domain, role) {
+ const presentSources = sources.filter(Boolean);
+
+ if (presentSources.length === 0) {
+ return {};
+ }
+
+ // find source duplicates that don't need to be re-downloaded or re-saved
+ const existingSourceItems = await knex('media').whereIn('source', presentSources.flat().map(source => source.src || source));
+ const { source: existingSourceItemsBySource, hash: existingSourceItemsByHash } = groupItems(existingSourceItems);
+
+ // download media items from new sources
+ const fetchedItems = await fetchItems(presentSources, existingSourceItemsBySource, domain, role);
+ const { hash: fetchedItemsByHash } = groupItems(fetchedItems);
+
+ // find hash duplicates that don't need to be re-saved
+ const uniqueFetchedItems = Object.values(fetchedItemsByHash);
+ const existingHashItems = await knex('media').whereIn('hash', uniqueFetchedItems.map(item => item.hash));
+ const { hash: existingHashItemsByHash } = groupItems(existingHashItems);
+
+ // save new items to disk
+ const newItems = uniqueFetchedItems.filter(item => !existingHashItemsByHash[item.hash]);
+ const savedItems = await saveItems(newItems, domain, role);
+
+ // store new items in database
+ const curatedItemEntries = curateItemEntries(savedItems);
+ const storedItems = await knex('media').insert(curatedItemEntries).returning('*');
+ const { hash: storedItemsByHash } = groupItems(Array.isArray(storedItems) ? storedItems : []);
+
+ // accumulate existing and new items by source to be mapped onto releases
+ const itemsByHash = { ...existingSourceItemsByHash, ...existingHashItemsByHash, ...storedItemsByHash };
+ const itemsBySource = {
+ ...existingSourceItemsBySource,
+ ...fetchedItems.reduce((acc, item) => ({ ...acc, [item.source]: itemsByHash[item.hash] }), {}),
+ };
+
+ logger.info(`Stored ${fetchedItems.length} new ${domain} ${role}s`);
+
+ return itemsBySource;
+}
+
+function extractPrimaryItem(associations, targetId, role, primaryRole, primaryItemsByTargetId) {
+ if (!primaryRole) {
+ return { [role]: associations, [primaryRole]: null };
+ }
+
+ if (primaryItemsByTargetId[targetId]) {
+ const remainingAssociations = associations.filter(association => association.media_id !== primaryItemsByTargetId[targetId].media_id);
+
+ return { [role]: remainingAssociations, [primaryRole]: null };
+ }
+
+ return {
+ [role]: associations.slice(1),
+ [primaryRole]: associations.slice(0, 1)[0],
+ };
+}
+
+function associateTargetMedia(targetId, sources, mediaBySource, domain, role, primaryRole, primaryItemsByTargetId) {
+ if (!sources) return { [role]: null, [primaryRole]: null };
+
+ const associations = sources
+ .filter(Boolean)
+ .map((source) => {
+ const mediaItem = Array.isArray(source)
+ ? source.reduce((acc, sourceX) => acc || mediaBySource[sourceX.src || sourceX], null)
+ : mediaBySource[source.src || source];
+
+ return mediaItem && { [`${domain}_id`]: targetId, media_id: mediaItem.id };
+ })
+ .filter(Boolean);
+
+ logger.info(`Associating ${associations.length} ${role}s to ${domain} ${targetId}`);
+
+ return extractPrimaryItem(associations, targetId, role, primaryRole, primaryItemsByTargetId);
+}
+
+async function associateMedia(sourcesByTargetId, mediaBySource, domain, role, primaryRole) {
+ const primaryItems = primaryRole ? await knex(`${domain}s_${primaryRole}s`).whereIn(`${domain}_id`, Object.keys(sourcesByTargetId)) : [];
+ const primaryItemsByTargetId = primaryItems.reduce((acc, item) => ({ ...acc, [item[`${domain}_id`]]: item }), {});
+
+ const associationsPerTarget = await Promise.map(Object.entries(sourcesByTargetId), ([targetId, sources]) => associateTargetMedia(targetId, sources, mediaBySource, domain, role, primaryRole, primaryItemsByTargetId));
+
+ const associations = associationsPerTarget.map(association => association[role]).flat().filter(Boolean);
+ const primaryAssociations = associationsPerTarget.map(association => association[primaryRole]).filter(Boolean);
+
+ return Promise.all([
+ (associations.length > 0 && knex.raw(`${knex(`${domain}s_${role}s`).insert(associations).toString()} ON CONFLICT DO NOTHING`)),
+ (primaryAssociations.length > 0 && knex.raw(`${knex(`${domain}s_${primaryRole}s`).insert(primaryAssociations).toString()} ON CONFLICT DO NOTHING`)),
+ ]);
}
module.exports = {
- storeReleaseMedia,
+ pluckItems,
+ storeMedia,
+ associateMedia,
};
diff --git a/src/releases.js b/src/releases.js
index c7253115d..b395d58a5 100644
--- a/src/releases.js
+++ b/src/releases.js
@@ -10,20 +10,11 @@ const argv = require('./argv');
const whereOr = require('./utils/where-or');
const { associateTags } = require('./tags');
const { associateActors, scrapeBasicActors } = require('./actors');
-/*
const {
- createMediaDirectory,
- storePhotos,
- // storeReleasePhotos,
- storeTrailer,
- storeReleaseMedia,
+ pluckItems,
+ storeMedia,
+ associateMedia,
} = require('./media');
-*/
-const {
- createMediaDirectory,
- storePhotos,
- storeTrailer,
-} = require('./media_legacy');
const { fetchSites, findSiteByUrl } = require('./sites');
const slugify = require('./utils/slugify');
@@ -337,55 +328,39 @@ function accumulateMovies(releases) {
}
async function storeReleaseAssets(releases) {
- // await storeReleasePhotos(releases);
+ const releasePostersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.poster] }), {});
+ const releaseCoversById = releases.reduce((acc, release) => ({ ...acc, [release.id]: release.covers }), {});
+ const releaseTrailersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.trailer] }), {});
+ const releaseTeasersById = releases.reduce((acc, release) => ({ ...acc, [release.id]: [release.teaser] }), {});
+ const releasePhotosById = releases.reduce((acc, release) => ({
+ ...acc,
+ [release.id]: pluckItems(release.photos),
+ }), {});
- // return storeReleaseMedia(releases);
+ const [posters, covers] = await Promise.all([
+ storeMedia(Object.values(releasePostersById).flat(), 'release', 'poster'),
+ storeMedia(Object.values(releaseCoversById).flat(), 'release', 'cover'),
+ ]);
- await Promise.map(releases, async (release) => {
- const subpath = `${release.site.network.slug}/${release.site.slug}/${release.id}/`;
- const identifier = `"${release.title}" (${release.id})`;
+ // ensure posters are available before fetching supplementary media
+ await Promise.all([
+ associateMedia(releasePostersById, posters, 'release', 'poster'),
+ associateMedia(releaseCoversById, covers, 'release', 'cover'),
+ ]);
- try {
- await createMediaDirectory('releases', subpath);
+ const photos = await storeMedia(Object.values(releasePhotosById).flat(), 'release', 'photo');
+ await associateMedia(releasePhotosById, photos, 'release', 'photo');
- // don't use Promise.all to prevent concurrency issues with duplicate detection
- if (release.poster) {
- await storePhotos([release.poster], {
- role: 'poster',
- targetId: release.id,
- subpath,
- }, identifier);
- }
+ // videos take a long time, fetch last
+ const [trailers, teasers] = await Promise.all([
+ storeMedia(Object.values(releaseTrailersById).flat(), 'release', 'trailer'),
+ storeMedia(Object.values(releaseTeasersById).flat(), 'release', 'teaser'),
+ ]);
- await storePhotos(release.photos, {
- targetId: release.id,
- subpath,
- primaryRole: release.poster ? null : 'poster',
- }, identifier);
-
- await storePhotos(release.covers, {
- role: 'cover',
- targetId: release.id,
- subpath,
- }, identifier);
-
- await storeTrailer(release.trailer, {
- targetId: release.id,
- subpath,
- role: 'trailer',
- }, identifier);
-
- await storeTrailer(release.teaser, {
- targetId: release.id,
- subpath,
- role: 'teaser',
- }, identifier);
- } catch (error) {
- logger.error(error.message);
- }
- }, {
- concurrency: 10,
- });
+ await Promise.all([
+ associateMedia(releaseTrailersById, trailers, 'release', 'trailer'),
+ associateMedia(releaseTeasersById, teasers, 'release', 'teaser'),
+ ]);
}
async function storeRelease(release) {
@@ -453,6 +428,8 @@ async function storeReleases(releases) {
concurrency: 10,
}).filter(release => release);
+ logger.info(`Stored ${storedReleases.length} new releases`);
+
const actors = accumulateActors(storedReleases);
const movies = accumulateMovies(storedReleases);
diff --git a/src/scrapers/dogfart.js b/src/scrapers/dogfart.js
index e807ad170..a3d4f17b0 100644
--- a/src/scrapers/dogfart.js
+++ b/src/scrapers/dogfart.js
@@ -47,7 +47,7 @@ function scrapeLatest(html, site) {
const actors = title.split(/[,&]|\band\b/).map(actor => actor.trim());
const poster = `https:${element.querySelector('img').src}`;
- const trailer = sceneLinkElement.dataset.preview_clip_url;
+ const teaser = sceneLinkElement.dataset.preview_clip_url;
return [
...acc,
@@ -57,8 +57,8 @@ function scrapeLatest(html, site) {
title,
actors,
poster,
- trailer: {
- src: trailer,
+ teaser: {
+ src: teaser,
},
site,
},
@@ -99,7 +99,7 @@ async function scrapeScene(html, url, site) {
const lastPhotosUrl = Array.from(document.querySelectorAll('.pagination a')).slice(-1)[0].href;
const photos = await getPhotos(`${origin}${pathname}${lastPhotosUrl}`, site, url);
- const stars = Math.floor(Number(document.querySelector('span[itemprop="average"]').textContent) / 2);
+ const stars = Math.floor(Number(document.querySelector('span[itemprop="average"]')?.textContent || document.querySelector('span[itemprop="ratingValue"]')?.textContent) / 2);
const tags = Array.from(document.querySelectorAll('.scene-details .categories a')).map(({ textContent }) => textContent);
return {
diff --git a/src/scrapers/mindgeek.js b/src/scrapers/mindgeek.js
index 1fd8526d2..fac69001a 100644
--- a/src/scrapers/mindgeek.js
+++ b/src/scrapers/mindgeek.js
@@ -36,35 +36,33 @@ function scrapeLatestX(data, site) {
return null;
}
- const { id: entryId, title, description } = data;
- const hostname = site.parameters?.native ? site.url : site.network.url;
- const url = `${hostname}/scene/${entryId}/`;
- const date = new Date(data.dateReleased);
- const actors = data.actors.map(actor => ({ name: actor.name, gender: actor.gender }));
-
- const tags = data.tags.map(tag => tag.name);
-
- const [poster, ...photos] = getThumbs(data);
- const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
- const duration = data.videos.mediabook && data.videos.mediabook.length;
-
- return {
- url,
- entryId,
- title,
- description,
- actors,
- tags,
- duration,
- poster,
- photos,
- trailer: trailer && {
- src: trailer.urls.view,
- quality: parseInt(trailer.format, 10),
- },
- date,
- site,
+ const release = {
+ entryId: data.id,
+ title: data.title,
+ description: data.description,
};
+
+ const hostname = site.parameters?.native ? site.url : site.network.url;
+
+ release.url = `${hostname}/scene/${release.entryId}/`;
+ release.date = new Date(data.dateReleased);
+ release.actors = data.actors.map(actor => ({ name: actor.name, gender: actor.gender }));
+
+ release.tags = data.tags.map(tag => tag.name);
+
+ release.duration = data.videos.mediabook?.length;
+ [release.poster, ...release.photos] = getThumbs(data);
+
+ const teaserSources = data.videos.mediabook?.files;
+
+ if (teaserSources) {
+ release.teaser = Object.values(teaserSources).map(teaser => ({
+ src: teaser.urls.view,
+ quality: parseInt(teaser.format, 10),
+ }));
+ }
+
+ return release;
}
async function scrapeLatest(items, site) {
@@ -89,12 +87,13 @@ function scrapeScene(data, url, _site, networkName) {
[release.poster, ...release.photos] = getThumbs(data);
- const trailer = data.videos.mediabook && (data.videos.mediabook.files['720p'] || data.videos.mediabook.files['320p']);
- if (trailer) {
- release.trailer = {
- src: trailer.urls.view,
- quality: parseInt(trailer.format, 10),
- };
+ const teaserSources = data.videos.mediabook?.files;
+
+ if (teaserSources) {
+ release.teaser = Object.values(teaserSources).map(teaser => ({
+ src: teaser.urls.view,
+ quality: parseInt(teaser.format, 10),
+ }));
}
const siteName = data.collections[0]?.name || data.brand;