From ad7874649fd1929e84c02e339639651d713528ee Mon Sep 17 00:00:00 2001 From: Niels Simenon Date: Thu, 2 Apr 2020 01:10:50 +0200 Subject: [PATCH] Added self hash filtering to media module. Moved Girl Girl back to Jules Jordan. --- assets/components/releases/banner.vue | 8 +-- assets/components/tile/tag.vue | 4 +- migrations/20190325001339_releases.js | 3 + .../network.png => julesjordan/girlgirl.png} | Bin .../misc/girl-girl_favicon.png} | Bin seeds/01_networks.js | 5 -- seeds/02_sites.js | 16 ++--- src/media.js | 61 +++++++++++------- 8 files changed, 54 insertions(+), 43 deletions(-) rename public/img/logos/{girlgirl/network.png => julesjordan/girlgirl.png} (100%) rename public/img/logos/{girlgirl/favicon.png => julesjordan/misc/girl-girl_favicon.png} (100%) diff --git a/assets/components/releases/banner.vue b/assets/components/releases/banner.vue index 4f1a7ad6..bf187a11 100644 --- a/assets/components/releases/banner.vue +++ b/assets/components/releases/banner.vue @@ -117,12 +117,12 @@ export default { @import 'theme'; .banner { - background: $empty; + background: var(--empty); flex-shrink: 0; white-space: nowrap; overflow-x: auto; scrollbar-width: none; - box-shadow: 0 0 3px $shadow; + box-shadow: 0 0 3px var(--shadow); font-size: 0; &::-webkit-scrollbar { @@ -150,7 +150,7 @@ export default { .icon { width: 1.5rem; height: 1.5rem; - fill: $highlight; + fill: var(--lighten); } &.playing { @@ -162,7 +162,7 @@ export default { opacity: 1; .icon { - fill: $highlight-strong; + fill: var(--lighten-strong); } } } diff --git a/assets/components/tile/tag.vue b/assets/components/tile/tag.vue index ece05ebd..44cbaffe 100644 --- a/assets/components/tile/tag.vue +++ b/assets/components/tile/tag.vue @@ -42,13 +42,12 @@ export default { background: var(--profile); display: flex; flex-direction: column; - align-items: left; justify-content: flex-end; box-sizing: border-box; position: relative; text-align: center; text-decoration: none; - box-shadow: inset 0 0 3px var(--darken); + box-shadow: 0 0 3px var(--darken-weak); } .poster { @@ -56,7 +55,6 @@ export default { height: 16rem; object-fit: cover; object-position: 50% 100%; - box-shadow: 0 0 1px var(--darken); } .title { diff --git a/migrations/20190325001339_releases.js b/migrations/20190325001339_releases.js index 2512a8b9..367233c0 100644 --- a/migrations/20190325001339_releases.js +++ b/migrations/20190325001339_releases.js @@ -296,6 +296,9 @@ exports.up = knex => Promise.resolve() table.string('piercings'); table.string('tattoos'); + table.datetime('updated_at') + .defaultTo(knex.fn.now()); + table.datetime('created_at') .defaultTo(knex.fn.now()); })) diff --git a/public/img/logos/girlgirl/network.png b/public/img/logos/julesjordan/girlgirl.png similarity index 100% rename from public/img/logos/girlgirl/network.png rename to public/img/logos/julesjordan/girlgirl.png diff --git a/public/img/logos/girlgirl/favicon.png b/public/img/logos/julesjordan/misc/girl-girl_favicon.png similarity index 100% rename from public/img/logos/girlgirl/favicon.png rename to public/img/logos/julesjordan/misc/girl-girl_favicon.png diff --git a/seeds/01_networks.js b/seeds/01_networks.js index 906a6715..7df534be 100644 --- a/seeds/01_networks.js +++ b/seeds/01_networks.js @@ -196,11 +196,6 @@ const networks = [ description: 'Girlsway.com has the best lesbian porn videos online! The hottest pornstars & first time lesbians in real girl on girl sex, tribbing, squirting & pussy licking action right HERE!', parent: 'gamma', }, - { - slug: 'girlgirl', - name: 'Girl Girl', - url: 'https://www.girlgirl.com', - }, { slug: 'hussiepass', name: 'Hussie Pass', diff --git a/seeds/02_sites.js b/seeds/02_sites.js index fe2c1f59..cd746f19 100644 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -2115,15 +2115,6 @@ const sites = [ tags: ['facefucking', 'blowjob'], network: 'fullpornnetwork', }, - // GIRL GIRL - { - slug: 'girlgirl', - name: 'Girl Girl', - url: 'https://www.girlgirl.com', - tags: ['lesbian'], - parameters: { independent: true }, - network: 'girlgirl', - }, // GIRLSWAY { slug: 'girlsway', @@ -2635,6 +2626,13 @@ const sites = [ url: 'https://www.manuelferrara.com', network: 'julesjordan', }, + { + slug: 'girlgirl', + name: 'Girl Girl', + url: 'https://www.girlgirl.com', + tags: ['lesbian'], + network: 'julesjordan', + }, // KELLY MADISON MEDIA { slug: 'teenfidelity', diff --git a/src/media.js b/src/media.js index ae64c50c..d7d625f2 100644 --- a/src/media.js +++ b/src/media.js @@ -1,7 +1,7 @@ 'use strict'; const config = require('config'); -const util = require('util'); +// const util = require('util'); const Promise = require('bluebird'); const fs = require('fs').promises; const path = require('path'); @@ -177,10 +177,7 @@ async function findSourceDuplicates(baseMedias) { const existingSourceMediaByUrl = itemsByKey(existingSourceMedia, 'source'); const existingExtractMediaByUrl = itemsByKey(existingExtractMedia, 'source_page'); - return { - existingSourceMediaByUrl, - existingExtractMediaByUrl, - }; + return [existingSourceMediaByUrl, existingExtractMediaByUrl]; } async function findHashDuplicates(medias) { @@ -189,16 +186,37 @@ async function findHashDuplicates(medias) { const existingHashMediaEntries = await knex('media').whereIn('hash', hashes); const existingHashMediaEntriesByHash = itemsByKey(existingHashMediaEntries, 'hash'); - const uniqueHashMedia = medias.filter(media => !media.entry && !existingHashMediaEntriesByHash[media.meta?.hash]); + const uniqueHashMedias = medias.filter(media => !media.entry && !existingHashMediaEntriesByHash[media.meta?.hash]); - const existingHashMedia = medias + const { selfDuplicateMedias, selfUniqueMediasByHash } = uniqueHashMedias.reduce((acc, media) => { + if (acc.selfUniqueMediasByHash[media.meta.hash]) { + acc.selfDuplicateMedias.push({ + ...media, + use: acc.selfUniqueMediasByHash[media.meta.hash].id, + }); + + return acc; + } + + acc.selfUniqueMediasByHash[media.meta.hash] = media; + + return acc; + }, { + selfDuplicateMedias: [], + selfUniqueMediasByHash: {}, + }); + + const selfUniqueHashMedias = Object.values(selfUniqueMediasByHash); + + const existingHashMedias = medias .filter(media => existingHashMediaEntriesByHash[media.entry?.hash || media.meta?.hash]) .map(media => ({ ...media, entry: existingHashMediaEntriesByHash[media.entry?.hash || media.meta?.hash], - })); + })) + .concat(selfDuplicateMedias); - return { uniqueHashMedia, existingHashMedia }; + return [selfUniqueHashMedias, existingHashMedias]; } async function extractSource(baseSource, { existingExtractMediaByUrl }) { @@ -337,7 +355,7 @@ async function fetchSource(source) { return attempt(1); } -async function trySource(baseSource, existingMedias, baseMedia, baseSourceIndex) { +async function trySource(baseSource, existingMedias) { // catch error and try the next source const extractedSource = await extractSource(baseSource, existingMedias); const existingSourceMedia = existingMedias.existingSourceMediaByUrl[extractedSource.src]; @@ -350,7 +368,7 @@ async function trySource(baseSource, existingMedias, baseMedia, baseSourceIndex) } if (existingSourceMedia) { - logger.silly(`Media source URL already in database, skipping ${baseSource.url}`); + logger.silly(`Media source URL already in database, skipping ${baseSource.src}`); // media entry found by source URL, don't fetch return { @@ -359,7 +377,7 @@ async function trySource(baseSource, existingMedias, baseMedia, baseSourceIndex) }; } - return fetchSource(extractedSource, baseMedia, baseSourceIndex, 1); + return fetchSource(extractedSource); } async function fetchMedia(baseMedia, existingMedias) { @@ -420,21 +438,21 @@ function curateMediaEntry(media, index) { } async function storeMedias(baseMedias) { - const { existingSourceMediaByUrl, existingExtractMediaByUrl } = await findSourceDuplicates(baseMedias); + const [existingSourceMediaByUrl, existingExtractMediaByUrl] = await findSourceDuplicates(baseMedias); const savedMedias = await Promise.map( baseMedias, async baseMedia => fetchMedia(baseMedia, { existingSourceMediaByUrl, existingExtractMediaByUrl }), ); - const { uniqueHashMedia, existingHashMedia } = await findHashDuplicates(savedMedias); + const [uniqueHashMedias, existingHashMedias] = await findHashDuplicates(savedMedias); - const newMediaWithEntries = uniqueHashMedia.map((media, index) => curateMediaEntry(media, index)); - const newMediaEntries = newMediaWithEntries.filter(media => !media.newEntry).map(media => media.entry); + const newMediaWithEntries = uniqueHashMedias.map((media, index) => curateMediaEntry(media, index)); + const newMediaEntries = newMediaWithEntries.filter(media => media.newEntry).map(media => media.entry); await knex('media').insert(newMediaEntries); - return [...newMediaWithEntries, ...existingHashMedia]; + return [...newMediaWithEntries, ...existingHashMedias]; } async function associateReleaseMedia(releases) { @@ -445,12 +463,13 @@ async function associateReleaseMedia(releases) { // TODO: internal duplicate filtering // TODO: media count limits // TODO: catch errors + // TODO: stage by role const baseMediasByReleaseId = releases.reduce((acc, release) => ({ ...acc, [release.id]: [ ...(argv.images && argv.poster ? toBaseMedias([release.poster], 'posters') : []), - ...(argv.images && argv.photos ? toBaseMedias(release.photos, 'photos').slice(0, 5) : []), + ...(argv.images && argv.photos ? toBaseMedias(release.photos, 'photos') : []), ...(argv.videos && argv.trailer ? toBaseMedias([release.trailer], 'trailers') : []), ...(argv.videos && argv.teaser ? toBaseMedias([release.teaser], 'teasers') : []), ], @@ -473,17 +492,15 @@ async function associateReleaseMedia(releases) { acc[media.role].push({ release_id: releaseId, - media_id: media.entry.id, + media_id: media.use || media.entry.id, }); }); return acc; }, {}); - console.log(util.inspect(associationsByRole, null, null)); - await Promise.all(Object.entries(associationsByRole) - .map(async ([role, associations]) => knex(`releases_${role}`).insert(associations))); + .map(async ([role, associations]) => knex.raw(`${knex(`releases_${role}`).insert(associations)} ON CONFLICT DO NOTHING`))); } module.exports = {