Preserving Jules Jordan base photos in deep scrape.

seeds/01_networks.js

@@ -475,6 +475,11 @@ const networks = [
 			forceDeep: true, // Mile High Media has movie and series information not available in the latest updates API
 		},
 	},
+	{
+		slug: 'modelmedia',
+		name: 'Model Media',
+		url: 'https://www.modelmediacash.com',
+	},
 	{
 		slug: 'mofos',
 		name: 'MOFOS',

seeds/02_sites.js

@@ -5976,6 +5976,21 @@ const sites = [
 		},
 		parent: 'milehighmedia',
 	},
+	// MODEL MEDIA
+	{
+		slug: 'delphine',
+		name: 'Delphine Films',
+		url: 'https://www.delphinefilms.com',
+		independent: true,
+		parent: 'modelmedia',
+	},
+	{
+		slug: 'asiam',
+		name: 'AsiaM',
+		url: 'https://www.modelmediaasia.com',
+		independent: true,
+		parent: 'modelmedia',
+	},
 	// MOFOS
 	{
 		slug: 'girlsgonepink',

src/argv.js

@@ -29,6 +29,12 @@ function interpretAfter(after, ignoreIfEmpty = false) {
 
 const { argv } = yargs
 	.command('npm start')
+	.command('update', 'fetch latest updates', () => {
+		console.log('UPDATING!');
+	})
+	.command('scene', 'fetch scene by URL', () => {
+		console.log('SCENE!');
+	})
 	.option('server', {
 		describe: 'Start web server',
 		type: 'boolean',

src/media.js

@@ -279,6 +279,7 @@ async function findSourceDuplicates(baseMedias) {
 	const [existingSourceMedia, existingExtractMedia] = await Promise.all([
 		// may try to check thousands of URLs at once, don't pass all of them to a single query
 		chunk(sourceUrls).reduce(async (chain, sourceUrlsChunk) => {
+			console.log(sourceUrlsChunk);
 			const accUrls = await chain;
 			const existingUrls = await knex('media').whereIn('source', sourceUrlsChunk);
 
@@ -930,7 +931,7 @@ async function associateReleaseMedia(releases, type = 'release') {
 				logger.error(util.inspect(error.entries, null, null, { color: true }));
 			}
 
-			logger.error(`Failed to store ${type} ${role}: ${error.message} (${error.detail || 'no detail'}`);
+			logger.error(`Failed to store ${type} ${role}: ${error.message} (${error.detail || 'no detail'})`);
 		}
 	}, Promise.resolve());
 }

src/scrapers/delphine.js

@@ -0,0 +1,85 @@
+'use strict';
+
+const unprint = require('unprint');
+
+const http = require('../utils/http');
+const qu = require('../utils/qu');
+
+function scrapeAll(scenes) {
+	return scenes.map(({ query }) => {
+		const release = {};
+
+		release.title = query.content('.video-title div');
+
+		console.log(release);
+
+		return release;
+	});
+}
+
+async function fetchLatest(channel, page) {
+	const session = http.session();
+
+	await http.get(channel.url, { session });
+	await http.post(`${channel.url}/adult_confirmation_and_accept_cookie`, null, { session });
+	const checkRes = await http.get(`${channel.url}/check_adult_confirmation_and_accept_cookie`, { session });
+
+	const res = await http.get(`${channel.url}/videos?sort=published_at&page=${page}`, { session });
+
+	// const res = await http.get(`${channel.url}/videos?sort=published_at&page=${page}`, { selectAll: '.row a[video-id]' });
+
+	console.log(checkRes.body);
+	console.log(res.body);
+
+	if (res.ok) {
+		return scrapeAll(res.context, channel);
+	}
+
+	return res.status;
+}
+
+/*
+
+    function confirmAdultCookie() {
+        // console.log('confirmed Adult');
+        $('#adult_confirmation').modal('hide');
+
+        var adultCookieRequest = $.ajax({
+            xhrFields: {
+                withCredentials: true
+            },
+            type: "POST",
+            url: "https://www.delphinefilms.com/adult_confirmation_and_accept_cookie"
+        });
+
+        $.when(adultCookieRequest).done(function() {
+            // console.log('Both requests completed successfully');
+            waitForSessionUpdateAndRefresh();
+        });
+
+        return false;
+    }
+
+    function waitForSessionUpdateAndRefresh() {
+        var interval = setInterval(function() {
+            $.ajax({
+                type: "GET",
+                url: "https://www.delphinefilms.com/check_adult_confirmation_and_accept_cookie",
+                success: function(response) {
+                    // console.log(response)
+                    if (response === "1") {
+                        // console.log('Session updated: Adult confirmed and cookie accepted');
+                        clearInterval(interval);
+                        location.reload();
+                    } else {
+                        // console.log('Session not updated: Adult not confirmed or cookie not accepted');
+                    }
+                }
+            });
+        }, 100);
+    }
+*/
+
+module.exports = {
+	fetchLatest,
+};

src/scrapers/julesjordan.js

@@ -51,28 +51,16 @@ function scrapeAll(scenes, site, entryIdFromTitle) {
 				const prefixedSrc = qu.prefixUrl(src, site.url);
 
 				if (src) {
-					return [
+					return Array.from(new Set([
-						{
+						prefixedSrc.replace(/.jpg$/, '-full.jpg'),
-							src: prefixedSrc.replace(/.jpg$/, '-full.jpg'),
+						prefixedSrc.replace(/-1x.jpg$/, '-4x.jpg'),
-							referer: site.url,
+						prefixedSrc.replace(/-1x.jpg$/, '-2x.jpg'),
-							verifyType: 'image', // sometimes returns 200 OK with text/html instead of 403
+						prefixedSrc,
-						},
+					])).map((source) => ({
-						{
+						src: source,
-							src: prefixedSrc.replace(/-1x.jpg$/, '-4x.jpg'),
 						referer: site.url,
 						verifyType: 'image',
-						},
+					}));
-						{
-							src: prefixedSrc.replace(/-1x.jpg$/, '-2x.jpg'),
-							referer: site.url,
-							verifyType: 'image',
-						},
-						{
-							src: prefixedSrc,
-							referer: site.url,
-							verifyType: 'image',
-						},
-					];
 				}
 
 				return null;
@@ -209,7 +197,21 @@ async function scrapeScene({ html, query }, context) {
 	if (argv.jjFullPhotos) {
 		release.photos = getPhotos(query, release, context);
 	} else {
-		release.photos = query.imgs('#images img');
+		// base release photos are usually better, but deep photos have additional thumbs
+		// the filenames are not chronological, so sorting after appending only worsens the mix
+		release.photos = [
+			...context.baseRelease?.photos?.map((sources) => sources.at(-1).src) || [],
+			...query.imgs('#images img'),
+		].map((source) => Array.from(new Set([
+			source.replace(/.jpg$/, '-full.jpg'),
+			source.replace(/-1x.jpg$/, '-4x.jpg'),
+			source.replace(/-1x.jpg$/, '-2x.jpg'),
+			source,
+		])).map((fallbackSource) => ({
+			src: fallbackSource,
+			referer: context.entity.url,
+			verifyType: 'image',
+		})));
 	}
 
 	if (query.exists('.update_dvds a')) {

src/scrapers/scrapers.js

@@ -15,6 +15,7 @@ const cherrypimps = require('./cherrypimps');
 const cumlouder = require('./cumlouder');
 const czechav = require('./czechav');
 const ddfnetwork = require('./ddfnetwork');
+const delphine = require('./delphine');
 const dorcel = require('./dorcel');
 const fabulouscash = require('./fabulouscash');
 const famedigital = require('./famedigital');
@@ -94,6 +95,7 @@ const scrapers = {
 		cumlouder,
 		czechav,
 		pornworld: ddfnetwork,
+		delphine,
 		dorcel,
 		elegantangel: adultempire,
 		famedigital,