Added movie tile. Fixed actor header. Larger breakpoint for nav menu.

src/app.js

@@ -8,7 +8,7 @@ const initServer = require('./web/server');
 const knex = require('./knex');
 const fetchUpdates = require('./updates');
 const { fetchScenes, fetchMovies } = require('./deep');
-const { storeReleases, updateReleasesSearch } = require('./store-releases');
+const { storeReleases, storeMovies, updateReleasesSearch } = require('./store-releases');
 const { scrapeActors } = require('./actors');
 const getFileEntries = require('./utils/file-entries');
 
@@ -31,14 +31,14 @@ async function init() {
 	const updateBaseScenes = (argv.all || argv.channels || argv.networks || argv.movies) && await fetchUpdates();
 
 	const scenesFromFile = argv.scenesFile && await getFileEntries(argv.scenesFile);
-	const sceneUrls = (argv.scenes || []).concat(scenesFromFile || []);
+	const sceneUrls = (argv.scene || []).concat(scenesFromFile || []);
 
 	const deepScenes = argv.deep
 		? await fetchScenes([...(sceneUrls), ...(updateBaseScenes || []), ...(actorBaseScenes || [])])
 		: [...(updateBaseScenes || []), ...(actorBaseScenes || [])];
 
-	const sceneMovies = deepScenes && argv.sceneMovies && deepScenes.map(scene => scene.movie).filter(Boolean);
-	const deepMovies = await fetchMovies([...(argv.movies || []), ...(sceneMovies || [])]);
+	const sceneMovies = deepScenes && argv.movie && deepScenes.map(scene => scene.movie).filter(Boolean);
+	const deepMovies = await fetchMovies([...(argv.movie || []), ...(sceneMovies || [])]);
 
 	if (argv.inspect) {
 		console.log(util.inspect(deepScenes));
@@ -46,10 +46,15 @@ async function init() {
 	}
 
 	if (argv.save) {
-		await storeReleases([
-			...(deepScenes || []),
-			...(deepMovies || []),
-		]);
+		if (deepScenes.length > 0) {
+			await storeReleases(deepScenes);
+		}
+
+		console.log(deepMovies);
+
+		if (deepMovies.length > 0) {
+			await storeMovies(deepMovies);
+		}
 	}
 
 	knex.destroy();

src/argv.js

@@ -25,10 +25,6 @@ const { argv } = yargs
 		type: 'array',
 		alias: 'channel',
 	})
-	.option('movies', {
-		describe: 'Scrape movies from channels',
-		type: 'array',
-	})
 	.option('actors', {
 		describe: 'Scrape actors by name or slug',
 		type: 'array',
@@ -61,19 +57,18 @@ const { argv } = yargs
 		alias: 'with-profiles',
 		default: false,
 	})
-	.option('scenes', {
+	.option('scene', {
 		describe: 'Scrape scene info from URL',
 		type: 'array',
-		alias: 'scene',
 	})
-	.option('scenes-file', {
+	.option('scene-file', {
 		describe: 'Scrape scene info from URLs in a file',
 		type: 'string',
+		alias: 'scenes-file',
 	})
 	.option('movie', {
 		describe: 'Scrape movie info from URL',
 		type: 'array',
-		alias: 'movies',
 	})
 	.option('sources', {
 		describe: 'Use these scrapers for actor data',
@@ -88,12 +83,17 @@ const { argv } = yargs
 	.option('latest', {
 		describe: 'Scrape latest releases if available',
 		type: 'boolean',
-		default: true,
+		default: false,
 	})
 	.option('upcoming', {
 		describe: 'Scrape upcoming releases if available',
 		type: 'boolean',
-		default: true,
+		default: false,
+	})
+	.option('movies', {
+		describe: 'Scrape movies from channels',
+		type: 'boolean',
+		default: false,
 	})
 	.option('force', {
 		describe: 'Don\'t ignore duplicates, update existing entries',

src/media.js

@@ -612,7 +612,7 @@ async function storeMedias(baseMedias) {
 	return [...newMediaWithEntries, ...existingHashMedias];
 }
 
-async function associateReleaseMedia(releases) {
+async function associateReleaseMedia(releases, type = 'releases') {
 	if (!argv.media) {
 		return;
 	}
@@ -670,7 +670,7 @@ async function associateReleaseMedia(releases) {
 			.filter(Boolean);
 
 		if (associations.length > 0) {
-			await knex.raw(`${knex(`releases_${role}`).insert(associations)} ON CONFLICT DO NOTHING`);
+			await knex.raw(`${knex(`${type}_${role}`).insert(associations)} ON CONFLICT DO NOTHING`);
 		}
 	}, Promise.resolve());
 }

src/scrapers/julesjordan.js

@@ -276,13 +276,13 @@ async function scrapeScene({ html, qu }, url, site, include) {
 	return release;
 }
 
-function scrapeMovie({ el, qu }, url, site) {
+function scrapeMovie({ el, query }, url, site) {
 	const movie = { url, site };
 
-	movie.entryId = qu.q('.dvd_details_overview .rating_box').dataset.id;
-	movie.title = qu.q('.title_bar span', true);
-	movie.covers = qu.urls('#dvd-cover-flip > a');
-	movie.channel = slugify(qu.q('.update_date a', true), '');
+	movie.entryId = query.q('.rating_box').dataset.id;
+	movie.title = query.q('.title_bar span', true);
+	movie.covers = query.urls('#dvd-cover-flip > a');
+	movie.channel = slugify(query.q('.update_date a', true), '');
 
 	// movie.releases = Array.from(document.querySelectorAll('.cell.dvd_info > a'), el => el.href);
 	const sceneQus = ctxa(el, '.dvd_details');

src/store-releases.js

@@ -12,7 +12,7 @@ const { associateReleaseTags } = require('./tags');
 const { curateEntity } = require('./entities');
 const { associateReleaseMedia } = require('./media');
 
-function curateReleaseEntry(release, batchId, existingRelease) {
+function curateReleaseEntry(release, batchId, existingRelease, type = 'scene') {
 	const slugBase = release.title
 		|| (release.actors?.length && `${release.entity.slug} ${release.actors.map(actor => actor.name).join(' ')}`)
 		|| (release.date && `${release.entity.slug} ${formatDate(release.date, 'YYYY MM DD')}`)
@@ -23,19 +23,18 @@ function curateReleaseEntry(release, batchId, existingRelease) {
 		limit: config.titleSlugLength,
 	});
 
+	console.log(release);
+
 	const curatedRelease = {
 		title: release.title,
 		entry_id: release.entryId || null,
 		entity_id: release.entity.id,
 		studio_id: release.studio?.id || null,
-		shoot_id: release.shootId || null,
 		url: release.url,
 		date: Number(release.date) ? release.date : null,
-		production_date: Number(release.productionDate) ? release.productionDate : null,
 		date_precision: release.datePrecision,
 		slug,
 		description: release.description,
-		duration: release.duration,
 		comment: release.comment,
 		// director: release.director,
 		// likes: release.rating && release.rating.likes,
@@ -46,6 +45,12 @@ function curateReleaseEntry(release, batchId, existingRelease) {
 		updated_batch_id: batchId,
 	};
 
+	if (type === 'scene') {
+		curatedRelease.shoot_id = release.shootId || null;
+		curatedRelease.productionDate = Number(release.productionDate) ? release.productionDate : null;
+		curatedRelease.duration = release.duration;
+	}
+
 	if (!existingRelease && !release.id) {
 		curatedRelease.created_batch_id = batchId;
 	}
@@ -252,7 +257,24 @@ async function storeReleases(releases) {
 	return releasesWithId;
 }
 
+async function storeMovies(movies) {
+	const [batchId] = await knex('batches').insert({ comment: null }).returning('id');
+
+	console.log(movies);
+
+	const curatedMovieEntries = movies.map(release => curateReleaseEntry(release, batchId, null, 'movie'));
+	console.log(curatedMovieEntries);
+	const storedMovies = await knex.batchInsert('movies', curatedMovieEntries).returning('*');
+
+	const moviesWithId = attachReleaseIds(movies, storedMovies);
+
+	await associateReleaseMedia(moviesWithId, 'movies');
+
+	return storedMovies;
+}
+
 module.exports = {
 	storeReleases,
+	storeMovies,
 	updateReleasesSearch,
 };

src/updates.js

@@ -161,6 +161,21 @@ async function scrapeUpcomingReleases(scraper, entity, preData) {
 	return [];
 }
 
+async function scrapeMovies(scraper, entity) {
+	if (!scraper.fetchMovies) {
+		return [];
+	}
+
+	try {
+		// return await scrapeReleases(scraper, entity, preData, true);
+		return await scraper.fetchMovies(entity);
+	} catch (error) {
+		logger.warn(`Failed to scrape movies for '${entity.slug}' (${entity.parent?.slug}): ${error.message}`);
+	}
+
+	return [];
+}
+
 async function scrapeChannelReleases(scraper, channelEntity, preData) {
 	const [latestReleases, upcomingReleases] = await Promise.all([
 		argv.latest
@@ -169,12 +184,11 @@ async function scrapeChannelReleases(scraper, channelEntity, preData) {
 		argv.upcoming
 			? scrapeUpcomingReleases(scraper, channelEntity, preData)
 			: [],
+		argv.movies
+			? scrapeMovies(scraper, channelEntity, preData)
+			: [],
 	]);
 
-	if (scraper.fetchMovies) {
-		await scraper.fetchMovies(channelEntity);
-	}
-
 	logger.info(`Fetching ${latestReleases.length} latest and ${upcomingReleases.length} upcoming updates for '${channelEntity.name}' (${channelEntity.parent?.name})`);
 
 	return [...latestReleases, ...upcomingReleases];