diff --git a/public/img/logos/asiam/asiam.png b/public/img/logos/asiam/asiam.png new file mode 100644 index 00000000..a832258d Binary files /dev/null and b/public/img/logos/asiam/asiam.png differ diff --git a/public/img/logos/asiam/favicon.png b/public/img/logos/asiam/favicon.png new file mode 100644 index 00000000..33aed759 Binary files /dev/null and b/public/img/logos/asiam/favicon.png differ diff --git a/public/img/logos/asiam/favicon_dark.png b/public/img/logos/asiam/favicon_dark.png new file mode 100644 index 00000000..e28965a0 Binary files /dev/null and b/public/img/logos/asiam/favicon_dark.png differ diff --git a/public/img/logos/asiam/favicon_light.png b/public/img/logos/asiam/favicon_light.png new file mode 100644 index 00000000..9315632f Binary files /dev/null and b/public/img/logos/asiam/favicon_light.png differ diff --git a/public/img/logos/asiam/lazy/asiam.png b/public/img/logos/asiam/lazy/asiam.png new file mode 100644 index 00000000..6b6d63b8 Binary files /dev/null and b/public/img/logos/asiam/lazy/asiam.png differ diff --git a/public/img/logos/asiam/lazy/favicon.png b/public/img/logos/asiam/lazy/favicon.png new file mode 100644 index 00000000..7a68f26c Binary files /dev/null and b/public/img/logos/asiam/lazy/favicon.png differ diff --git a/public/img/logos/asiam/lazy/favicon_dark.png b/public/img/logos/asiam/lazy/favicon_dark.png new file mode 100644 index 00000000..5fa8e6ac Binary files /dev/null and b/public/img/logos/asiam/lazy/favicon_dark.png differ diff --git a/public/img/logos/asiam/lazy/favicon_light.png b/public/img/logos/asiam/lazy/favicon_light.png new file mode 100644 index 00000000..ff7a1c17 Binary files /dev/null and b/public/img/logos/asiam/lazy/favicon_light.png differ diff --git a/public/img/logos/asiam/lazy/network.png b/public/img/logos/asiam/lazy/network.png new file mode 100644 index 00000000..024fb5f8 Binary files /dev/null and b/public/img/logos/asiam/lazy/network.png differ diff --git a/public/img/logos/asiam/misc/asia-m-mm.png b/public/img/logos/asiam/misc/asia-m-mm.png new file mode 100644 index 00000000..ef228044 Binary files /dev/null and b/public/img/logos/asiam/misc/asia-m-mm.png differ diff --git a/public/img/logos/asiam/network.png b/public/img/logos/asiam/network.png new file mode 100644 index 00000000..21f88bf6 Binary files /dev/null and b/public/img/logos/asiam/network.png differ diff --git a/public/img/logos/asiam/thumbs/asiam.png b/public/img/logos/asiam/thumbs/asiam.png new file mode 100644 index 00000000..36df9f9b Binary files /dev/null and b/public/img/logos/asiam/thumbs/asiam.png differ diff --git a/public/img/logos/asiam/thumbs/favicon.png b/public/img/logos/asiam/thumbs/favicon.png new file mode 100644 index 00000000..7a68f26c Binary files /dev/null and b/public/img/logos/asiam/thumbs/favicon.png differ diff --git a/public/img/logos/asiam/thumbs/favicon_dark.png b/public/img/logos/asiam/thumbs/favicon_dark.png new file mode 100644 index 00000000..5fa8e6ac Binary files /dev/null and b/public/img/logos/asiam/thumbs/favicon_dark.png differ diff --git a/public/img/logos/asiam/thumbs/favicon_light.png b/public/img/logos/asiam/thumbs/favicon_light.png new file mode 100644 index 00000000..ff7a1c17 Binary files /dev/null and b/public/img/logos/asiam/thumbs/favicon_light.png differ diff --git a/public/img/logos/asiam/thumbs/network.png b/public/img/logos/asiam/thumbs/network.png new file mode 100644 index 00000000..a36ecc89 Binary files /dev/null and b/public/img/logos/asiam/thumbs/network.png differ diff --git a/public/img/logos/delphine/delphine.png b/public/img/logos/delphine/delphine.png new file mode 100644 index 00000000..f1a8c74f Binary files /dev/null and b/public/img/logos/delphine/delphine.png differ diff --git a/public/img/logos/delphine/favicon.png b/public/img/logos/delphine/favicon.png new file mode 100644 index 00000000..1a27051d Binary files /dev/null and b/public/img/logos/delphine/favicon.png differ diff --git a/public/img/logos/delphine/favicon_dark.png b/public/img/logos/delphine/favicon_dark.png new file mode 100644 index 00000000..70f97f8a Binary files /dev/null and b/public/img/logos/delphine/favicon_dark.png differ diff --git a/public/img/logos/delphine/favicon_light.png b/public/img/logos/delphine/favicon_light.png new file mode 100644 index 00000000..cd8d9ef9 Binary files /dev/null and b/public/img/logos/delphine/favicon_light.png differ diff --git a/public/img/logos/delphine/lazy/delphine.png b/public/img/logos/delphine/lazy/delphine.png new file mode 100644 index 00000000..9186f87b Binary files /dev/null and b/public/img/logos/delphine/lazy/delphine.png differ diff --git a/public/img/logos/delphine/lazy/favicon.png b/public/img/logos/delphine/lazy/favicon.png new file mode 100644 index 00000000..60f0c039 Binary files /dev/null and b/public/img/logos/delphine/lazy/favicon.png differ diff --git a/public/img/logos/delphine/lazy/favicon_dark.png b/public/img/logos/delphine/lazy/favicon_dark.png new file mode 100644 index 00000000..93d8d065 Binary files /dev/null and b/public/img/logos/delphine/lazy/favicon_dark.png differ diff --git a/public/img/logos/delphine/lazy/favicon_light.png b/public/img/logos/delphine/lazy/favicon_light.png new file mode 100644 index 00000000..76e0fc2c Binary files /dev/null and b/public/img/logos/delphine/lazy/favicon_light.png differ diff --git a/public/img/logos/delphine/lazy/network.png b/public/img/logos/delphine/lazy/network.png new file mode 100644 index 00000000..7dacec43 Binary files /dev/null and b/public/img/logos/delphine/lazy/network.png differ diff --git a/public/img/logos/delphine/misc/delphine-mm.png b/public/img/logos/delphine/misc/delphine-mm.png new file mode 100644 index 00000000..ae7f3547 Binary files /dev/null and b/public/img/logos/delphine/misc/delphine-mm.png differ diff --git a/public/img/logos/delphine/network.png b/public/img/logos/delphine/network.png new file mode 100644 index 00000000..c9c123fe Binary files /dev/null and b/public/img/logos/delphine/network.png differ diff --git a/public/img/logos/delphine/thumbs/delphine.png b/public/img/logos/delphine/thumbs/delphine.png new file mode 100644 index 00000000..5df52461 Binary files /dev/null and b/public/img/logos/delphine/thumbs/delphine.png differ diff --git a/public/img/logos/delphine/thumbs/favicon.png b/public/img/logos/delphine/thumbs/favicon.png new file mode 100644 index 00000000..172731e7 Binary files /dev/null and b/public/img/logos/delphine/thumbs/favicon.png differ diff --git a/public/img/logos/delphine/thumbs/favicon_dark.png b/public/img/logos/delphine/thumbs/favicon_dark.png new file mode 100644 index 00000000..39cc2d56 Binary files /dev/null and b/public/img/logos/delphine/thumbs/favicon_dark.png differ diff --git a/public/img/logos/delphine/thumbs/favicon_light.png b/public/img/logos/delphine/thumbs/favicon_light.png new file mode 100644 index 00000000..420a6c80 Binary files /dev/null and b/public/img/logos/delphine/thumbs/favicon_light.png differ diff --git a/public/img/logos/delphine/thumbs/network.png b/public/img/logos/delphine/thumbs/network.png new file mode 100644 index 00000000..5d10a956 Binary files /dev/null and b/public/img/logos/delphine/thumbs/network.png differ diff --git a/public/img/logos/modelmedia/favicon.png b/public/img/logos/modelmedia/favicon.png new file mode 100644 index 00000000..46d461d3 Binary files /dev/null and b/public/img/logos/modelmedia/favicon.png differ diff --git a/public/img/logos/modelmedia/favicon_dark.png b/public/img/logos/modelmedia/favicon_dark.png new file mode 100644 index 00000000..ceb54c7d Binary files /dev/null and b/public/img/logos/modelmedia/favicon_dark.png differ diff --git a/public/img/logos/modelmedia/favicon_light.png b/public/img/logos/modelmedia/favicon_light.png new file mode 100644 index 00000000..c5bf27c7 Binary files /dev/null and b/public/img/logos/modelmedia/favicon_light.png differ diff --git a/public/img/logos/modelmedia/lazy/favicon.png b/public/img/logos/modelmedia/lazy/favicon.png new file mode 100644 index 00000000..e782c4ae Binary files /dev/null and b/public/img/logos/modelmedia/lazy/favicon.png differ diff --git a/public/img/logos/modelmedia/lazy/favicon_dark.png b/public/img/logos/modelmedia/lazy/favicon_dark.png new file mode 100644 index 00000000..daac1226 Binary files /dev/null and b/public/img/logos/modelmedia/lazy/favicon_dark.png differ diff --git a/public/img/logos/modelmedia/lazy/favicon_light.png b/public/img/logos/modelmedia/lazy/favicon_light.png new file mode 100644 index 00000000..323db95d Binary files /dev/null and b/public/img/logos/modelmedia/lazy/favicon_light.png differ diff --git a/public/img/logos/modelmedia/lazy/modelmedia.png b/public/img/logos/modelmedia/lazy/modelmedia.png new file mode 100644 index 00000000..57b74a7e Binary files /dev/null and b/public/img/logos/modelmedia/lazy/modelmedia.png differ diff --git a/public/img/logos/modelmedia/lazy/network.png b/public/img/logos/modelmedia/lazy/network.png new file mode 100644 index 00000000..2344b24d Binary files /dev/null and b/public/img/logos/modelmedia/lazy/network.png differ diff --git a/public/img/logos/modelmedia/misc/model-media-cash.png b/public/img/logos/modelmedia/misc/model-media-cash.png new file mode 100644 index 00000000..53036078 Binary files /dev/null and b/public/img/logos/modelmedia/misc/model-media-cash.png differ diff --git a/public/img/logos/modelmedia/misc/model-media.png b/public/img/logos/modelmedia/misc/model-media.png new file mode 100644 index 00000000..962c825f Binary files /dev/null and b/public/img/logos/modelmedia/misc/model-media.png differ diff --git a/public/img/logos/modelmedia/modelmedia.png b/public/img/logos/modelmedia/modelmedia.png new file mode 100644 index 00000000..d38d8805 Binary files /dev/null and b/public/img/logos/modelmedia/modelmedia.png differ diff --git a/public/img/logos/modelmedia/network.png b/public/img/logos/modelmedia/network.png new file mode 100644 index 00000000..dd278634 Binary files /dev/null and b/public/img/logos/modelmedia/network.png differ diff --git a/public/img/logos/modelmedia/thumbs/favicon.png b/public/img/logos/modelmedia/thumbs/favicon.png new file mode 100644 index 00000000..e782c4ae Binary files /dev/null and b/public/img/logos/modelmedia/thumbs/favicon.png differ diff --git a/public/img/logos/modelmedia/thumbs/favicon_dark.png b/public/img/logos/modelmedia/thumbs/favicon_dark.png new file mode 100644 index 00000000..daac1226 Binary files /dev/null and b/public/img/logos/modelmedia/thumbs/favicon_dark.png differ diff --git a/public/img/logos/modelmedia/thumbs/favicon_light.png b/public/img/logos/modelmedia/thumbs/favicon_light.png new file mode 100644 index 00000000..323db95d Binary files /dev/null and b/public/img/logos/modelmedia/thumbs/favicon_light.png differ diff --git a/public/img/logos/modelmedia/thumbs/modelmedia.png b/public/img/logos/modelmedia/thumbs/modelmedia.png new file mode 100644 index 00000000..04ec9108 Binary files /dev/null and b/public/img/logos/modelmedia/thumbs/modelmedia.png differ diff --git a/public/img/logos/modelmedia/thumbs/network.png b/public/img/logos/modelmedia/thumbs/network.png new file mode 100644 index 00000000..d66e43d3 Binary files /dev/null and b/public/img/logos/modelmedia/thumbs/network.png differ diff --git a/seeds/01_networks.js b/seeds/01_networks.js index 2f9f721d..ead60d73 100755 --- a/seeds/01_networks.js +++ b/seeds/01_networks.js @@ -475,6 +475,11 @@ const networks = [ forceDeep: true, // Mile High Media has movie and series information not available in the latest updates API }, }, + { + slug: 'modelmedia', + name: 'Model Media', + url: 'https://www.modelmediacash.com', + }, { slug: 'mofos', name: 'MOFOS', diff --git a/seeds/02_sites.js b/seeds/02_sites.js index 59cc0c76..f19c490b 100755 --- a/seeds/02_sites.js +++ b/seeds/02_sites.js @@ -5976,6 +5976,21 @@ const sites = [ }, parent: 'milehighmedia', }, + // MODEL MEDIA + { + slug: 'delphine', + name: 'Delphine Films', + url: 'https://www.delphinefilms.com', + independent: true, + parent: 'modelmedia', + }, + { + slug: 'asiam', + name: 'AsiaM', + url: 'https://www.modelmediaasia.com', + independent: true, + parent: 'modelmedia', + }, // MOFOS { slug: 'girlsgonepink', diff --git a/src/argv.js b/src/argv.js index 66e39b2a..b444da43 100755 --- a/src/argv.js +++ b/src/argv.js @@ -29,6 +29,12 @@ function interpretAfter(after, ignoreIfEmpty = false) { const { argv } = yargs .command('npm start') + .command('update', 'fetch latest updates', () => { + console.log('UPDATING!'); + }) + .command('scene', 'fetch scene by URL', () => { + console.log('SCENE!'); + }) .option('server', { describe: 'Start web server', type: 'boolean', diff --git a/src/media.js b/src/media.js index cddc2635..417950ff 100755 --- a/src/media.js +++ b/src/media.js @@ -279,6 +279,7 @@ async function findSourceDuplicates(baseMedias) { const [existingSourceMedia, existingExtractMedia] = await Promise.all([ // may try to check thousands of URLs at once, don't pass all of them to a single query chunk(sourceUrls).reduce(async (chain, sourceUrlsChunk) => { + console.log(sourceUrlsChunk); const accUrls = await chain; const existingUrls = await knex('media').whereIn('source', sourceUrlsChunk); @@ -930,7 +931,7 @@ async function associateReleaseMedia(releases, type = 'release') { logger.error(util.inspect(error.entries, null, null, { color: true })); } - logger.error(`Failed to store ${type} ${role}: ${error.message} (${error.detail || 'no detail'}`); + logger.error(`Failed to store ${type} ${role}: ${error.message} (${error.detail || 'no detail'})`); } }, Promise.resolve()); } diff --git a/src/scrapers/delphine.js b/src/scrapers/delphine.js new file mode 100644 index 00000000..1cc29adf --- /dev/null +++ b/src/scrapers/delphine.js @@ -0,0 +1,85 @@ +'use strict'; + +const unprint = require('unprint'); + +const http = require('../utils/http'); +const qu = require('../utils/qu'); + +function scrapeAll(scenes) { + return scenes.map(({ query }) => { + const release = {}; + + release.title = query.content('.video-title div'); + + console.log(release); + + return release; + }); +} + +async function fetchLatest(channel, page) { + const session = http.session(); + + await http.get(channel.url, { session }); + await http.post(`${channel.url}/adult_confirmation_and_accept_cookie`, null, { session }); + const checkRes = await http.get(`${channel.url}/check_adult_confirmation_and_accept_cookie`, { session }); + + const res = await http.get(`${channel.url}/videos?sort=published_at&page=${page}`, { session }); + + // const res = await http.get(`${channel.url}/videos?sort=published_at&page=${page}`, { selectAll: '.row a[video-id]' }); + + console.log(checkRes.body); + console.log(res.body); + + if (res.ok) { + return scrapeAll(res.context, channel); + } + + return res.status; +} + +/* + + function confirmAdultCookie() { + // console.log('confirmed Adult'); + $('#adult_confirmation').modal('hide'); + + var adultCookieRequest = $.ajax({ + xhrFields: { + withCredentials: true + }, + type: "POST", + url: "https://www.delphinefilms.com/adult_confirmation_and_accept_cookie" + }); + + $.when(adultCookieRequest).done(function() { + // console.log('Both requests completed successfully'); + waitForSessionUpdateAndRefresh(); + }); + + return false; + } + + function waitForSessionUpdateAndRefresh() { + var interval = setInterval(function() { + $.ajax({ + type: "GET", + url: "https://www.delphinefilms.com/check_adult_confirmation_and_accept_cookie", + success: function(response) { + // console.log(response) + if (response === "1") { + // console.log('Session updated: Adult confirmed and cookie accepted'); + clearInterval(interval); + location.reload(); + } else { + // console.log('Session not updated: Adult not confirmed or cookie not accepted'); + } + } + }); + }, 100); + } +*/ + +module.exports = { + fetchLatest, +}; diff --git a/src/scrapers/julesjordan.js b/src/scrapers/julesjordan.js index fe46756b..584fb456 100755 --- a/src/scrapers/julesjordan.js +++ b/src/scrapers/julesjordan.js @@ -51,28 +51,16 @@ function scrapeAll(scenes, site, entryIdFromTitle) { const prefixedSrc = qu.prefixUrl(src, site.url); if (src) { - return [ - { - src: prefixedSrc.replace(/.jpg$/, '-full.jpg'), - referer: site.url, - verifyType: 'image', // sometimes returns 200 OK with text/html instead of 403 - }, - { - src: prefixedSrc.replace(/-1x.jpg$/, '-4x.jpg'), - referer: site.url, - verifyType: 'image', - }, - { - src: prefixedSrc.replace(/-1x.jpg$/, '-2x.jpg'), - referer: site.url, - verifyType: 'image', - }, - { - src: prefixedSrc, - referer: site.url, - verifyType: 'image', - }, - ]; + return Array.from(new Set([ + prefixedSrc.replace(/.jpg$/, '-full.jpg'), + prefixedSrc.replace(/-1x.jpg$/, '-4x.jpg'), + prefixedSrc.replace(/-1x.jpg$/, '-2x.jpg'), + prefixedSrc, + ])).map((source) => ({ + src: source, + referer: site.url, + verifyType: 'image', + })); } return null; @@ -209,7 +197,21 @@ async function scrapeScene({ html, query }, context) { if (argv.jjFullPhotos) { release.photos = getPhotos(query, release, context); } else { - release.photos = query.imgs('#images img'); + // base release photos are usually better, but deep photos have additional thumbs + // the filenames are not chronological, so sorting after appending only worsens the mix + release.photos = [ + ...context.baseRelease?.photos?.map((sources) => sources.at(-1).src) || [], + ...query.imgs('#images img'), + ].map((source) => Array.from(new Set([ + source.replace(/.jpg$/, '-full.jpg'), + source.replace(/-1x.jpg$/, '-4x.jpg'), + source.replace(/-1x.jpg$/, '-2x.jpg'), + source, + ])).map((fallbackSource) => ({ + src: fallbackSource, + referer: context.entity.url, + verifyType: 'image', + }))); } if (query.exists('.update_dvds a')) { diff --git a/src/scrapers/scrapers.js b/src/scrapers/scrapers.js index b075c73c..9ef6c590 100755 --- a/src/scrapers/scrapers.js +++ b/src/scrapers/scrapers.js @@ -15,6 +15,7 @@ const cherrypimps = require('./cherrypimps'); const cumlouder = require('./cumlouder'); const czechav = require('./czechav'); const ddfnetwork = require('./ddfnetwork'); +const delphine = require('./delphine'); const dorcel = require('./dorcel'); const fabulouscash = require('./fabulouscash'); const famedigital = require('./famedigital'); @@ -94,6 +95,7 @@ const scrapers = { cumlouder, czechav, pornworld: ddfnetwork, + delphine, dorcel, elegantangel: adultempire, famedigital,