diff --git a/assets/js/config/default.js b/assets/js/config/default.js index dc353d19..d303d2d1 100644 --- a/assets/js/config/default.js +++ b/assets/js/config/default.js @@ -12,6 +12,7 @@ export default { selectableTags: [ 'airtight', 'anal', + 'bdsm', 'blowbang', 'blowjob', 'creampie', diff --git a/migrations/20220330230122_stats.js b/migrations/20220330230122_stats.js new file mode 100644 index 00000000..fde6a0cd --- /dev/null +++ b/migrations/20220330230122_stats.js @@ -0,0 +1,25 @@ +exports.up = async (knex) => knex.raw(` + CREATE MATERIALIZED VIEW entities_stats + AS + WITH RECURSIVE relations AS ( + SELECT entities.id, entities.parent_id, count(releases.id) AS releases_count, count(releases.id) AS total_count + FROM entities + LEFT JOIN releases ON releases.entity_id = entities.id + GROUP BY entities.id + + UNION ALL + + SELECT entities.id AS entity_id, count(releases.id) AS releases_count, count(releases.id) + relations.total_count AS total_count + FROM entities + INNER JOIN relations ON relations.id = entities.parent_id + LEFT JOIN releases ON releases.entity_id = entities.id + GROUP BY entities.id + ) + + SELECT relations.id AS entity_id, relations.releases_count + FROM relations; +`); + +exports.down = async (knex) => knex.raw(` + DROP MATERIALIZED VIEW entities_stats; +`); diff --git a/public/img/logos/pervcity/dpdiva.png b/public/img/logos/pervcity/dpdiva.png new file mode 100644 index 00000000..2f8285df Binary files /dev/null and b/public/img/logos/pervcity/dpdiva.png differ diff --git a/public/img/logos/pervcity/lazy/analoverdose.png b/public/img/logos/pervcity/lazy/analoverdose.png index 7e78cec0..3ad41999 100644 Binary files a/public/img/logos/pervcity/lazy/analoverdose.png and b/public/img/logos/pervcity/lazy/analoverdose.png differ diff --git a/public/img/logos/pervcity/lazy/bangingbeauties.png b/public/img/logos/pervcity/lazy/bangingbeauties.png index 8783d541..3637d528 100644 Binary files a/public/img/logos/pervcity/lazy/bangingbeauties.png and b/public/img/logos/pervcity/lazy/bangingbeauties.png differ diff --git a/public/img/logos/pervcity/lazy/chocolatebjs.png b/public/img/logos/pervcity/lazy/chocolatebjs.png index a4ec7b57..881e2489 100644 Binary files a/public/img/logos/pervcity/lazy/chocolatebjs.png and b/public/img/logos/pervcity/lazy/chocolatebjs.png differ diff --git a/public/img/logos/pervcity/lazy/dpdiva.png b/public/img/logos/pervcity/lazy/dpdiva.png new file mode 100644 index 00000000..cfd887c5 Binary files /dev/null and b/public/img/logos/pervcity/lazy/dpdiva.png differ diff --git a/public/img/logos/pervcity/lazy/favicon.png b/public/img/logos/pervcity/lazy/favicon.png index e72cb4e6..c4496e46 100644 Binary files a/public/img/logos/pervcity/lazy/favicon.png and b/public/img/logos/pervcity/lazy/favicon.png differ diff --git a/public/img/logos/pervcity/lazy/favicon_dark.png b/public/img/logos/pervcity/lazy/favicon_dark.png new file mode 100644 index 00000000..35814c37 Binary files /dev/null and b/public/img/logos/pervcity/lazy/favicon_dark.png differ diff --git a/public/img/logos/pervcity/lazy/favicon_light.png b/public/img/logos/pervcity/lazy/favicon_light.png new file mode 100644 index 00000000..35814c37 Binary files /dev/null and b/public/img/logos/pervcity/lazy/favicon_light.png differ diff --git a/public/img/logos/pervcity/lazy/network.png b/public/img/logos/pervcity/lazy/network.png index 3be745a5..f56493d5 100644 Binary files a/public/img/logos/pervcity/lazy/network.png and b/public/img/logos/pervcity/lazy/network.png differ diff --git a/public/img/logos/pervcity/lazy/oraloverdose.png b/public/img/logos/pervcity/lazy/oraloverdose.png index ab6af5b0..c9f45f4f 100644 Binary files a/public/img/logos/pervcity/lazy/oraloverdose.png and b/public/img/logos/pervcity/lazy/oraloverdose.png differ diff --git a/public/img/logos/pervcity/lazy/pervcity.png b/public/img/logos/pervcity/lazy/pervcity.png index db7ac364..a9b3933c 100644 Binary files a/public/img/logos/pervcity/lazy/pervcity.png and b/public/img/logos/pervcity/lazy/pervcity.png differ diff --git a/public/img/logos/pervcity/lazy/upherasshole.png b/public/img/logos/pervcity/lazy/upherasshole.png index 362256fe..6e6c4cdd 100644 Binary files a/public/img/logos/pervcity/lazy/upherasshole.png and b/public/img/logos/pervcity/lazy/upherasshole.png differ diff --git a/public/img/logos/pervcity/thumbs/analoverdose.png b/public/img/logos/pervcity/thumbs/analoverdose.png index 49151fe0..e3f5bb5e 100644 Binary files a/public/img/logos/pervcity/thumbs/analoverdose.png and b/public/img/logos/pervcity/thumbs/analoverdose.png differ diff --git a/public/img/logos/pervcity/thumbs/bangingbeauties.png b/public/img/logos/pervcity/thumbs/bangingbeauties.png index 33ec98f5..ce3e9ee6 100644 Binary files a/public/img/logos/pervcity/thumbs/bangingbeauties.png and b/public/img/logos/pervcity/thumbs/bangingbeauties.png differ diff --git a/public/img/logos/pervcity/thumbs/chocolatebjs.png b/public/img/logos/pervcity/thumbs/chocolatebjs.png index b5b4e24a..aa15ea49 100644 Binary files a/public/img/logos/pervcity/thumbs/chocolatebjs.png and b/public/img/logos/pervcity/thumbs/chocolatebjs.png differ diff --git a/public/img/logos/pervcity/thumbs/dpdiva.png b/public/img/logos/pervcity/thumbs/dpdiva.png new file mode 100644 index 00000000..3927138e Binary files /dev/null and b/public/img/logos/pervcity/thumbs/dpdiva.png differ diff --git a/public/img/logos/pervcity/thumbs/favicon.png b/public/img/logos/pervcity/thumbs/favicon.png index e72cb4e6..c4496e46 100644 Binary files a/public/img/logos/pervcity/thumbs/favicon.png and b/public/img/logos/pervcity/thumbs/favicon.png differ diff --git a/public/img/logos/pervcity/thumbs/favicon_dark.png b/public/img/logos/pervcity/thumbs/favicon_dark.png new file mode 100644 index 00000000..35814c37 Binary files /dev/null and b/public/img/logos/pervcity/thumbs/favicon_dark.png differ diff --git a/public/img/logos/pervcity/thumbs/favicon_light.png b/public/img/logos/pervcity/thumbs/favicon_light.png new file mode 100644 index 00000000..35814c37 Binary files /dev/null and b/public/img/logos/pervcity/thumbs/favicon_light.png differ diff --git a/public/img/logos/pervcity/thumbs/network.png b/public/img/logos/pervcity/thumbs/network.png index 97da214d..7d328d3e 100644 Binary files a/public/img/logos/pervcity/thumbs/network.png and b/public/img/logos/pervcity/thumbs/network.png differ diff --git a/public/img/logos/pervcity/thumbs/oraloverdose.png b/public/img/logos/pervcity/thumbs/oraloverdose.png index 8746b7ee..cad0fc7d 100644 Binary files a/public/img/logos/pervcity/thumbs/oraloverdose.png and b/public/img/logos/pervcity/thumbs/oraloverdose.png differ diff --git a/public/img/logos/pervcity/thumbs/pervcity.png b/public/img/logos/pervcity/thumbs/pervcity.png index 6ebf130b..7d328d3e 100644 Binary files a/public/img/logos/pervcity/thumbs/pervcity.png and b/public/img/logos/pervcity/thumbs/pervcity.png differ diff --git a/public/img/logos/pervcity/thumbs/upherasshole.png b/public/img/logos/pervcity/thumbs/upherasshole.png index c77e175e..0436600d 100644 Binary files a/public/img/logos/pervcity/thumbs/upherasshole.png and b/public/img/logos/pervcity/thumbs/upherasshole.png differ diff --git a/seeds/02_sites.js b/seeds/02_sites.js index a166e0b8..f04934ce 100644 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -6917,6 +6917,13 @@ const sites = [ tourId: 9, }, }, + { + slug: 'dpdiva', + name: 'DP Diva', + url: 'http://dpdiva.com', + parent: 'pervcity', + tags: ['dp', 'anal'], + }, // PIERRE WOODMAN { slug: 'woodmancastingx', diff --git a/src/app.js b/src/app.js index a8fdac83..27774b9b 100644 --- a/src/app.js +++ b/src/app.js @@ -85,23 +85,6 @@ async function startMemorySample(snapshotTriggers = []) { }, config.memorySampling.sampleDuration); } -async function startMemorySample() { - await inspector.heap.enable(); - await inspector.heap.startSampling(); - - // monitorMemory(); - - logger.info(`Start heap sampling, memory usage: ${process.memoryUsage.rss() / 1000000} MB`); - - setTimeout(async () => { - await stopMemorySample(); - - if (!done) { - await startMemorySample(); - } - }, 30000); -} - async function init() { try { if (argv.server) { diff --git a/src/media.js b/src/media.js index eaf8e6f0..a3043d02 100644 --- a/src/media.js +++ b/src/media.js @@ -21,6 +21,7 @@ const argv = require('./argv'); const knex = require('./knex'); const http = require('./utils/http'); const bulkInsert = require('./utils/bulk-insert'); +const chunk = require('./utils/chunk'); const { get } = require('./utils/qu'); const pipeline = util.promisify(stream.pipeline); @@ -63,10 +64,10 @@ function sampleMedias(medias, limit = argv.mediaLimit, preferLast = true) { ? chunks.slice(0, -1).concat(chunks.slice(-1).reverse()) : chunks; - const groupedMedias = lastPreferredChunks.map((chunk) => { + const groupedMedias = lastPreferredChunks.map((mediaChunk) => { // merge chunked medias into single media with grouped fallback priorities, // so the first sources of each media is preferred over all second sources, etc. - const sources = chunk + const sources = mediaChunk .reduce((accSources, media) => { media.sources.forEach((source, index) => { if (!accSources[index]) { @@ -82,8 +83,8 @@ function sampleMedias(medias, limit = argv.mediaLimit, preferLast = true) { .flat(); return { - id: chunk[0].id, - role: chunk[0].role, + id: mediaChunk[0].id, + role: mediaChunk[0].role, sources, }; }); @@ -235,22 +236,41 @@ async function findSourceDuplicates(baseMedias) { .filter(Boolean); const [existingSourceMedia, existingExtractMedia] = await Promise.all([ - knex('media').whereIn('source', sourceUrls), - knex('media').whereIn('source_page', extractUrls), + // my try to check thousands of URLs at once, don't pass all of them to a single query + chunk(sourceUrls).reduce(async (chain, sourceUrlsChunk) => { + const accUrls = await chain; + const existingUrls = await knex('media').whereIn('source', sourceUrlsChunk); + + return [...accUrls, ...existingUrls]; + }, []), + chunk(extractUrls).reduce(async (chain, extractUrlsChunk) => { + const accUrls = await chain; + const existingUrls = await knex('media').whereIn('source_page', extractUrlsChunk); + + return [...accUrls, ...existingUrls]; + }, []), ]); const existingSourceMediaByUrl = itemsByKey(existingSourceMedia, 'source'); const existingExtractMediaByUrl = itemsByKey(existingExtractMedia, 'source_page'); - return { existingSourceMediaByUrl, existingExtractMediaByUrl }; + return { + existingSourceMediaByUrl, + existingExtractMediaByUrl, + }; } async function findHashDuplicates(medias) { const hashes = medias.map((media) => media.meta?.hash || media.entry?.hash).filter(Boolean); - const existingHashMediaEntries = await knex('media').whereIn('hash', hashes); - const existingHashMediaEntriesByHash = itemsByKey(existingHashMediaEntries, 'hash'); + const existingHashMediaEntries = await chunk(hashes, 2).reduce(async (chain, hashesChunk) => { + const accHashes = await chain; + const existingHashes = await knex('media').whereIn('hash', hashesChunk); + return [...accHashes, ...existingHashes]; + }, []); + + const existingHashMediaEntriesByHash = itemsByKey(existingHashMediaEntries, 'hash'); const uniqueHashMedias = medias.filter((media) => !media.entry && !existingHashMediaEntriesByHash[media.meta?.hash]); const { selfDuplicateMedias, selfUniqueMediasByHash } = uniqueHashMedias.reduce((acc, media) => { @@ -600,11 +620,11 @@ async function fetchSource(source, baseMedia) { const hashStream = new stream.PassThrough(); let size = 0; - hashStream.on('data', (chunk) => { - size += chunk.length; + hashStream.on('data', (streamChunk) => { + size += streamChunk.length; if (hasherReady) { - hasher.write(chunk); + hasher.write(streamChunk); } });