diff --git a/public/img/logos/asiam/asiam.png b/public/img/logos/asiam/asiam.png new file mode 100644 index 000000000..a832258df Binary files /dev/null and b/public/img/logos/asiam/asiam.png differ diff --git a/public/img/logos/asiam/favicon.png b/public/img/logos/asiam/favicon.png new file mode 100644 index 000000000..33aed7593 Binary files /dev/null and b/public/img/logos/asiam/favicon.png differ diff --git a/public/img/logos/asiam/favicon_dark.png b/public/img/logos/asiam/favicon_dark.png new file mode 100644 index 000000000..e28965a09 Binary files /dev/null and b/public/img/logos/asiam/favicon_dark.png differ diff --git a/public/img/logos/asiam/favicon_light.png b/public/img/logos/asiam/favicon_light.png new file mode 100644 index 000000000..9315632f5 Binary files /dev/null and b/public/img/logos/asiam/favicon_light.png differ diff --git a/public/img/logos/asiam/lazy/asiam.png b/public/img/logos/asiam/lazy/asiam.png new file mode 100644 index 000000000..6b6d63b85 Binary files /dev/null and b/public/img/logos/asiam/lazy/asiam.png differ diff --git a/public/img/logos/asiam/lazy/favicon.png b/public/img/logos/asiam/lazy/favicon.png new file mode 100644 index 000000000..7a68f26c8 Binary files /dev/null and b/public/img/logos/asiam/lazy/favicon.png differ diff --git a/public/img/logos/asiam/lazy/favicon_dark.png b/public/img/logos/asiam/lazy/favicon_dark.png new file mode 100644 index 000000000..5fa8e6acc Binary files /dev/null and b/public/img/logos/asiam/lazy/favicon_dark.png differ diff --git a/public/img/logos/asiam/lazy/favicon_light.png b/public/img/logos/asiam/lazy/favicon_light.png new file mode 100644 index 000000000..ff7a1c179 Binary files /dev/null and b/public/img/logos/asiam/lazy/favicon_light.png differ diff --git a/public/img/logos/asiam/lazy/network.png b/public/img/logos/asiam/lazy/network.png new file mode 100644 index 000000000..024fb5f8c Binary files /dev/null and b/public/img/logos/asiam/lazy/network.png differ diff --git a/public/img/logos/asiam/misc/asia-m-mm.png b/public/img/logos/asiam/misc/asia-m-mm.png new file mode 100644 index 000000000..ef228044f Binary files /dev/null and b/public/img/logos/asiam/misc/asia-m-mm.png differ diff --git a/public/img/logos/asiam/network.png b/public/img/logos/asiam/network.png new file mode 100644 index 000000000..21f88bf6c Binary files /dev/null and b/public/img/logos/asiam/network.png differ diff --git a/public/img/logos/asiam/thumbs/asiam.png b/public/img/logos/asiam/thumbs/asiam.png new file mode 100644 index 000000000..36df9f9b9 Binary files /dev/null and b/public/img/logos/asiam/thumbs/asiam.png differ diff --git a/public/img/logos/asiam/thumbs/favicon.png b/public/img/logos/asiam/thumbs/favicon.png new file mode 100644 index 000000000..7a68f26c8 Binary files /dev/null and b/public/img/logos/asiam/thumbs/favicon.png differ diff --git a/public/img/logos/asiam/thumbs/favicon_dark.png b/public/img/logos/asiam/thumbs/favicon_dark.png new file mode 100644 index 000000000..5fa8e6acc Binary files /dev/null and b/public/img/logos/asiam/thumbs/favicon_dark.png differ diff --git a/public/img/logos/asiam/thumbs/favicon_light.png b/public/img/logos/asiam/thumbs/favicon_light.png new file mode 100644 index 000000000..ff7a1c179 Binary files /dev/null and b/public/img/logos/asiam/thumbs/favicon_light.png differ diff --git a/public/img/logos/asiam/thumbs/network.png b/public/img/logos/asiam/thumbs/network.png new file mode 100644 index 000000000..a36ecc897 Binary files /dev/null and b/public/img/logos/asiam/thumbs/network.png differ diff --git a/public/img/logos/delphine/delphine.png b/public/img/logos/delphine/delphine.png new file mode 100644 index 000000000..f1a8c74f5 Binary files /dev/null and b/public/img/logos/delphine/delphine.png differ diff --git a/public/img/logos/delphine/favicon.png b/public/img/logos/delphine/favicon.png new file mode 100644 index 000000000..1a27051d9 Binary files /dev/null and b/public/img/logos/delphine/favicon.png differ diff --git a/public/img/logos/delphine/favicon_dark.png b/public/img/logos/delphine/favicon_dark.png new file mode 100644 index 000000000..70f97f8af Binary files /dev/null and b/public/img/logos/delphine/favicon_dark.png differ diff --git a/public/img/logos/delphine/favicon_light.png b/public/img/logos/delphine/favicon_light.png new file mode 100644 index 000000000..cd8d9ef92 Binary files /dev/null and b/public/img/logos/delphine/favicon_light.png differ diff --git a/public/img/logos/delphine/lazy/delphine.png b/public/img/logos/delphine/lazy/delphine.png new file mode 100644 index 000000000..9186f87b1 Binary files /dev/null and b/public/img/logos/delphine/lazy/delphine.png differ diff --git a/public/img/logos/delphine/lazy/favicon.png b/public/img/logos/delphine/lazy/favicon.png new file mode 100644 index 000000000..60f0c039c Binary files /dev/null and b/public/img/logos/delphine/lazy/favicon.png differ diff --git a/public/img/logos/delphine/lazy/favicon_dark.png b/public/img/logos/delphine/lazy/favicon_dark.png new file mode 100644 index 000000000..93d8d0652 Binary files /dev/null and b/public/img/logos/delphine/lazy/favicon_dark.png differ diff --git a/public/img/logos/delphine/lazy/favicon_light.png b/public/img/logos/delphine/lazy/favicon_light.png new file mode 100644 index 000000000..76e0fc2c6 Binary files /dev/null and b/public/img/logos/delphine/lazy/favicon_light.png differ diff --git a/public/img/logos/delphine/lazy/network.png b/public/img/logos/delphine/lazy/network.png new file mode 100644 index 000000000..7dacec438 Binary files /dev/null and b/public/img/logos/delphine/lazy/network.png differ diff --git a/public/img/logos/delphine/misc/delphine-mm.png b/public/img/logos/delphine/misc/delphine-mm.png new file mode 100644 index 000000000..ae7f3547c Binary files /dev/null and b/public/img/logos/delphine/misc/delphine-mm.png differ diff --git a/public/img/logos/delphine/network.png b/public/img/logos/delphine/network.png new file mode 100644 index 000000000..c9c123fee Binary files /dev/null and b/public/img/logos/delphine/network.png differ diff --git a/public/img/logos/delphine/thumbs/delphine.png b/public/img/logos/delphine/thumbs/delphine.png new file mode 100644 index 000000000..5df524615 Binary files /dev/null and b/public/img/logos/delphine/thumbs/delphine.png differ diff --git a/public/img/logos/delphine/thumbs/favicon.png b/public/img/logos/delphine/thumbs/favicon.png new file mode 100644 index 000000000..172731e76 Binary files /dev/null and b/public/img/logos/delphine/thumbs/favicon.png differ diff --git a/public/img/logos/delphine/thumbs/favicon_dark.png b/public/img/logos/delphine/thumbs/favicon_dark.png new file mode 100644 index 000000000..39cc2d56d Binary files /dev/null and b/public/img/logos/delphine/thumbs/favicon_dark.png differ diff --git a/public/img/logos/delphine/thumbs/favicon_light.png b/public/img/logos/delphine/thumbs/favicon_light.png new file mode 100644 index 000000000..420a6c806 Binary files /dev/null and b/public/img/logos/delphine/thumbs/favicon_light.png differ diff --git a/public/img/logos/delphine/thumbs/network.png b/public/img/logos/delphine/thumbs/network.png new file mode 100644 index 000000000..5d10a956d Binary files /dev/null and b/public/img/logos/delphine/thumbs/network.png differ diff --git a/public/img/logos/modelmedia/favicon.png b/public/img/logos/modelmedia/favicon.png new file mode 100644 index 000000000..46d461d32 Binary files /dev/null and b/public/img/logos/modelmedia/favicon.png differ diff --git a/public/img/logos/modelmedia/favicon_dark.png b/public/img/logos/modelmedia/favicon_dark.png new file mode 100644 index 000000000..ceb54c7d7 Binary files /dev/null and b/public/img/logos/modelmedia/favicon_dark.png differ diff --git a/public/img/logos/modelmedia/favicon_light.png b/public/img/logos/modelmedia/favicon_light.png new file mode 100644 index 000000000..c5bf27c78 Binary files /dev/null and b/public/img/logos/modelmedia/favicon_light.png differ diff --git a/public/img/logos/modelmedia/lazy/favicon.png b/public/img/logos/modelmedia/lazy/favicon.png new file mode 100644 index 000000000..e782c4ae0 Binary files /dev/null and b/public/img/logos/modelmedia/lazy/favicon.png differ diff --git a/public/img/logos/modelmedia/lazy/favicon_dark.png b/public/img/logos/modelmedia/lazy/favicon_dark.png new file mode 100644 index 000000000..daac12261 Binary files /dev/null and b/public/img/logos/modelmedia/lazy/favicon_dark.png differ diff --git a/public/img/logos/modelmedia/lazy/favicon_light.png b/public/img/logos/modelmedia/lazy/favicon_light.png new file mode 100644 index 000000000..323db95d4 Binary files /dev/null and b/public/img/logos/modelmedia/lazy/favicon_light.png differ diff --git a/public/img/logos/modelmedia/lazy/modelmedia.png b/public/img/logos/modelmedia/lazy/modelmedia.png new file mode 100644 index 000000000..57b74a7e2 Binary files /dev/null and b/public/img/logos/modelmedia/lazy/modelmedia.png differ diff --git a/public/img/logos/modelmedia/lazy/network.png b/public/img/logos/modelmedia/lazy/network.png new file mode 100644 index 000000000..2344b24d6 Binary files /dev/null and b/public/img/logos/modelmedia/lazy/network.png differ diff --git a/public/img/logos/modelmedia/misc/model-media-cash.png b/public/img/logos/modelmedia/misc/model-media-cash.png new file mode 100644 index 000000000..53036078f Binary files /dev/null and b/public/img/logos/modelmedia/misc/model-media-cash.png differ diff --git a/public/img/logos/modelmedia/misc/model-media.png b/public/img/logos/modelmedia/misc/model-media.png new file mode 100644 index 000000000..962c825fa Binary files /dev/null and b/public/img/logos/modelmedia/misc/model-media.png differ diff --git a/public/img/logos/modelmedia/modelmedia.png b/public/img/logos/modelmedia/modelmedia.png new file mode 100644 index 000000000..d38d8805c Binary files /dev/null and b/public/img/logos/modelmedia/modelmedia.png differ diff --git a/public/img/logos/modelmedia/network.png b/public/img/logos/modelmedia/network.png new file mode 100644 index 000000000..dd278634f Binary files /dev/null and b/public/img/logos/modelmedia/network.png differ diff --git a/public/img/logos/modelmedia/thumbs/favicon.png b/public/img/logos/modelmedia/thumbs/favicon.png new file mode 100644 index 000000000..e782c4ae0 Binary files /dev/null and b/public/img/logos/modelmedia/thumbs/favicon.png differ diff --git a/public/img/logos/modelmedia/thumbs/favicon_dark.png b/public/img/logos/modelmedia/thumbs/favicon_dark.png new file mode 100644 index 000000000..daac12261 Binary files /dev/null and b/public/img/logos/modelmedia/thumbs/favicon_dark.png differ diff --git a/public/img/logos/modelmedia/thumbs/favicon_light.png b/public/img/logos/modelmedia/thumbs/favicon_light.png new file mode 100644 index 000000000..323db95d4 Binary files /dev/null and b/public/img/logos/modelmedia/thumbs/favicon_light.png differ diff --git a/public/img/logos/modelmedia/thumbs/modelmedia.png b/public/img/logos/modelmedia/thumbs/modelmedia.png new file mode 100644 index 000000000..04ec9108a Binary files /dev/null and b/public/img/logos/modelmedia/thumbs/modelmedia.png differ diff --git a/public/img/logos/modelmedia/thumbs/network.png b/public/img/logos/modelmedia/thumbs/network.png new file mode 100644 index 000000000..d66e43d35 Binary files /dev/null and b/public/img/logos/modelmedia/thumbs/network.png differ diff --git a/seeds/01_networks.js b/seeds/01_networks.js index 2f9f721d1..ead60d736 100755 --- a/seeds/01_networks.js +++ b/seeds/01_networks.js @@ -475,6 +475,11 @@ const networks = [ forceDeep: true, // Mile High Media has movie and series information not available in the latest updates API }, }, + { + slug: 'modelmedia', + name: 'Model Media', + url: 'https://www.modelmediacash.com', + }, { slug: 'mofos', name: 'MOFOS', diff --git a/seeds/02_sites.js b/seeds/02_sites.js index 59cc0c760..f19c490b2 100755 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -5976,6 +5976,21 @@ const sites = [ }, parent: 'milehighmedia', }, + // MODEL MEDIA + { + slug: 'delphine', + name: 'Delphine Films', + url: 'https://www.delphinefilms.com', + independent: true, + parent: 'modelmedia', + }, + { + slug: 'asiam', + name: 'AsiaM', + url: 'https://www.modelmediaasia.com', + independent: true, + parent: 'modelmedia', + }, // MOFOS { slug: 'girlsgonepink', diff --git a/src/argv.js b/src/argv.js index 66e39b2a6..b444da434 100755 --- a/src/argv.js +++ b/src/argv.js @@ -29,6 +29,12 @@ function interpretAfter(after, ignoreIfEmpty = false) { const { argv } = yargs .command('npm start') + .command('update', 'fetch latest updates', () => { + console.log('UPDATING!'); + }) + .command('scene', 'fetch scene by URL', () => { + console.log('SCENE!'); + }) .option('server', { describe: 'Start web server', type: 'boolean', diff --git a/src/media.js b/src/media.js index cddc2635d..417950ff5 100755 --- a/src/media.js +++ b/src/media.js @@ -279,6 +279,7 @@ async function findSourceDuplicates(baseMedias) { const [existingSourceMedia, existingExtractMedia] = await Promise.all([ // may try to check thousands of URLs at once, don't pass all of them to a single query chunk(sourceUrls).reduce(async (chain, sourceUrlsChunk) => { + console.log(sourceUrlsChunk); const accUrls = await chain; const existingUrls = await knex('media').whereIn('source', sourceUrlsChunk); @@ -930,7 +931,7 @@ async function associateReleaseMedia(releases, type = 'release') { logger.error(util.inspect(error.entries, null, null, { color: true })); } - logger.error(`Failed to store ${type} ${role}: ${error.message} (${error.detail || 'no detail'}`); + logger.error(`Failed to store ${type} ${role}: ${error.message} (${error.detail || 'no detail'})`); } }, Promise.resolve()); } diff --git a/src/scrapers/delphine.js b/src/scrapers/delphine.js new file mode 100644 index 000000000..1cc29adfa --- /dev/null +++ b/src/scrapers/delphine.js @@ -0,0 +1,85 @@ +'use strict'; + +const unprint = require('unprint'); + +const http = require('../utils/http'); +const qu = require('../utils/qu'); + +function scrapeAll(scenes) { + return scenes.map(({ query }) => { + const release = {}; + + release.title = query.content('.video-title div'); + + console.log(release); + + return release; + }); +} + +async function fetchLatest(channel, page) { + const session = http.session(); + + await http.get(channel.url, { session }); + await http.post(`${channel.url}/adult_confirmation_and_accept_cookie`, null, { session }); + const checkRes = await http.get(`${channel.url}/check_adult_confirmation_and_accept_cookie`, { session }); + + const res = await http.get(`${channel.url}/videos?sort=published_at&page=${page}`, { session }); + + // const res = await http.get(`${channel.url}/videos?sort=published_at&page=${page}`, { selectAll: '.row a[video-id]' }); + + console.log(checkRes.body); + console.log(res.body); + + if (res.ok) { + return scrapeAll(res.context, channel); + } + + return res.status; +} + +/* + + function confirmAdultCookie() { + // console.log('confirmed Adult'); + $('#adult_confirmation').modal('hide'); + + var adultCookieRequest = $.ajax({ + xhrFields: { + withCredentials: true + }, + type: "POST", + url: "https://www.delphinefilms.com/adult_confirmation_and_accept_cookie" + }); + + $.when(adultCookieRequest).done(function() { + // console.log('Both requests completed successfully'); + waitForSessionUpdateAndRefresh(); + }); + + return false; + } + + function waitForSessionUpdateAndRefresh() { + var interval = setInterval(function() { + $.ajax({ + type: "GET", + url: "https://www.delphinefilms.com/check_adult_confirmation_and_accept_cookie", + success: function(response) { + // console.log(response) + if (response === "1") { + // console.log('Session updated: Adult confirmed and cookie accepted'); + clearInterval(interval); + location.reload(); + } else { + // console.log('Session not updated: Adult not confirmed or cookie not accepted'); + } + } + }); + }, 100); + } +*/ + +module.exports = { + fetchLatest, +}; diff --git a/src/scrapers/julesjordan.js b/src/scrapers/julesjordan.js index fe46756b7..584fb456b 100755 --- a/src/scrapers/julesjordan.js +++ b/src/scrapers/julesjordan.js @@ -51,28 +51,16 @@ function scrapeAll(scenes, site, entryIdFromTitle) { const prefixedSrc = qu.prefixUrl(src, site.url); if (src) { - return [ - { - src: prefixedSrc.replace(/.jpg$/, '-full.jpg'), - referer: site.url, - verifyType: 'image', // sometimes returns 200 OK with text/html instead of 403 - }, - { - src: prefixedSrc.replace(/-1x.jpg$/, '-4x.jpg'), - referer: site.url, - verifyType: 'image', - }, - { - src: prefixedSrc.replace(/-1x.jpg$/, '-2x.jpg'), - referer: site.url, - verifyType: 'image', - }, - { - src: prefixedSrc, - referer: site.url, - verifyType: 'image', - }, - ]; + return Array.from(new Set([ + prefixedSrc.replace(/.jpg$/, '-full.jpg'), + prefixedSrc.replace(/-1x.jpg$/, '-4x.jpg'), + prefixedSrc.replace(/-1x.jpg$/, '-2x.jpg'), + prefixedSrc, + ])).map((source) => ({ + src: source, + referer: site.url, + verifyType: 'image', + })); } return null; @@ -209,7 +197,21 @@ async function scrapeScene({ html, query }, context) { if (argv.jjFullPhotos) { release.photos = getPhotos(query, release, context); } else { - release.photos = query.imgs('#images img'); + // base release photos are usually better, but deep photos have additional thumbs + // the filenames are not chronological, so sorting after appending only worsens the mix + release.photos = [ + ...context.baseRelease?.photos?.map((sources) => sources.at(-1).src) || [], + ...query.imgs('#images img'), + ].map((source) => Array.from(new Set([ + source.replace(/.jpg$/, '-full.jpg'), + source.replace(/-1x.jpg$/, '-4x.jpg'), + source.replace(/-1x.jpg$/, '-2x.jpg'), + source, + ])).map((fallbackSource) => ({ + src: fallbackSource, + referer: context.entity.url, + verifyType: 'image', + }))); } if (query.exists('.update_dvds a')) { diff --git a/src/scrapers/scrapers.js b/src/scrapers/scrapers.js index b075c73cb..9ef6c5901 100755 --- a/src/scrapers/scrapers.js +++ b/src/scrapers/scrapers.js @@ -15,6 +15,7 @@ const cherrypimps = require('./cherrypimps'); const cumlouder = require('./cumlouder'); const czechav = require('./czechav'); const ddfnetwork = require('./ddfnetwork'); +const delphine = require('./delphine'); const dorcel = require('./dorcel'); const fabulouscash = require('./fabulouscash'); const famedigital = require('./famedigital'); @@ -94,6 +95,7 @@ const scrapers = { cumlouder, czechav, pornworld: ddfnetwork, + delphine, dorcel, elegantangel: adultempire, famedigital,