Separated error log. Added search/summary update tool.

seeds/02_sites.js

@@ -8517,16 +8517,6 @@ const sites = [
 		independent: true,
 		parent: 'radical',
 	},
-	{
-		name: 'Got Filled',
-		slug: 'gotfilled',
-		url: 'https://gotfilled.com',
-		independent: true,
-		parent: 'radical',
-		parameters: {
-			layout: 'metadata',
-		},
-	},
 	{
 		name: 'Inserted',
 		slug: 'inserted',
@@ -8534,7 +8524,19 @@ const sites = [
 		independent: true,
 		parent: 'radical',
 		parameters: {
-			layout: 'metadata',
+			layout: 'api',
+			endpoint: 'QrQe_TF3broC5P80XTIbd',
+		},
+	},
+	{
+		name: 'Got Filled',
+		slug: 'gotfilled',
+		url: 'https://gotfilled.com',
+		independent: true,
+		parent: 'radical',
+		parameters: {
+			layout: 'api',
+			endpoint: 'nOpFJJgD_-c5PrBqecVXA',
 		},
 	},
 	{
@@ -8544,7 +8546,8 @@ const sites = [
 		independent: true,
 		parent: 'radical',
 		parameters: {
-			layout: 'metadata',
+			layout: 'api',
+			endpoint: 'fnkMPhO2Gd-XwWTZHyftg',
 		},
 	},
 	// REALITY KINGS

src/logger.js

@@ -29,9 +29,14 @@ function logger(filepath) {
 			}),
 			new winston.transports.DailyRotateFile({
 				datePattern: 'YYYY-MM-DD',
-				filename: path.join('log', '%DATE%.log'),
+				filename: path.join('log', 'combined_%DATE%.log'),
 				level: 'silly',
 			}),
+			new winston.transports.DailyRotateFile({
+				datePattern: 'YYYY-MM-DD',
+				filename: path.join('log', 'error_%DATE%.log'),
+				level: 'error',
+			}),
 		],
 	});
 }

src/scrapers/radical.js

@@ -15,7 +15,7 @@ function scrapeSceneMetadata(data, channel) {
 	release.description = data.description;
 
 	release.date = new Date(data.release_date);
-	release.duration = qu.durationToSeconds(data.videos_duration);
+	release.duration = data.seconds_duration || qu.durationToSeconds(data.videos_duration);
 
 	release.actors = data.models.map((model) => ({
 		entryId: model.id,
@@ -50,6 +50,22 @@ function scrapeAllMetadata(scenes, channel) {
 	return scenes.map((data) => scrapeSceneMetadata(data, channel));
 }
 
+function scrapeAllApi(scenes, channel) {
+	return scenes.map((data) => {
+		const release = {};
+
+		release.entryId = data.id;
+
+		release.title = data.title;
+		release.description = data.description;
+
+		console.log(data);
+		console.log(release);
+
+		return release;
+	});
+}
+
 function scrapeProfileMetadata(data, channel) {
 	const profile = {};
 
@@ -95,6 +111,16 @@ async function fetchLatestMetadata(channel, page = 1) {
 	return res.status;
 }
 
+async function fetchLatestApi(channel, page, { parameters }) {
+	const res = await http.get(`${channel.url}/_next/data/${parameters.endpoint}/videos.json?order_by=publish_date&sort_by=desc&per_page=30&page=${page}`);
+
+	if (res.ok) {
+		return scrapeAllApi(res.body.pageProps.contents.data, channel);
+	}
+
+	return res.status;
+}
+
 async function fetchSceneMetadata(url, channel) {
 	const res = await http.get(url, {
 		parse: true,
@@ -136,8 +162,12 @@ async function fetchProfileMetadata(actor, channel) {
 
 module.exports = {
 	metadata: {
+		// probably deprecated
 		fetchLatest: fetchLatestMetadata,
 		fetchScene: fetchSceneMetadata,
 		fetchProfile: fetchProfileMetadata,
 	},
+	api: {
+		fetchLatest: fetchLatestApi,
+	},
 };

src/tools/update-search.js

@@ -0,0 +1,12 @@
+'use strict';
+
+const { updateSceneSearch, updateMovieSearch } = require('../update-search');
+
+async function init() {
+	await updateSceneSearch();
+	await updateMovieSearch();
+
+	process.exit();
+}
+
+init();