Separated error log. Added search/summary update tool.

This commit is contained in:
DebaucheryLibrarian 2023-08-13 21:57:32 +02:00
parent d5806c3d31
commit c860bfebc1
4 changed files with 64 additions and 14 deletions

View File

@ -8517,16 +8517,6 @@ const sites = [
independent: true, independent: true,
parent: 'radical', parent: 'radical',
}, },
{
name: 'Got Filled',
slug: 'gotfilled',
url: 'https://gotfilled.com',
independent: true,
parent: 'radical',
parameters: {
layout: 'metadata',
},
},
{ {
name: 'Inserted', name: 'Inserted',
slug: 'inserted', slug: 'inserted',
@ -8534,7 +8524,19 @@ const sites = [
independent: true, independent: true,
parent: 'radical', parent: 'radical',
parameters: { parameters: {
layout: 'metadata', layout: 'api',
endpoint: 'QrQe_TF3broC5P80XTIbd',
},
},
{
name: 'Got Filled',
slug: 'gotfilled',
url: 'https://gotfilled.com',
independent: true,
parent: 'radical',
parameters: {
layout: 'api',
endpoint: 'nOpFJJgD_-c5PrBqecVXA',
}, },
}, },
{ {
@ -8544,7 +8546,8 @@ const sites = [
independent: true, independent: true,
parent: 'radical', parent: 'radical',
parameters: { parameters: {
layout: 'metadata', layout: 'api',
endpoint: 'fnkMPhO2Gd-XwWTZHyftg',
}, },
}, },
// REALITY KINGS // REALITY KINGS

View File

@ -29,9 +29,14 @@ function logger(filepath) {
}), }),
new winston.transports.DailyRotateFile({ new winston.transports.DailyRotateFile({
datePattern: 'YYYY-MM-DD', datePattern: 'YYYY-MM-DD',
filename: path.join('log', '%DATE%.log'), filename: path.join('log', 'combined_%DATE%.log'),
level: 'silly', level: 'silly',
}), }),
new winston.transports.DailyRotateFile({
datePattern: 'YYYY-MM-DD',
filename: path.join('log', 'error_%DATE%.log'),
level: 'error',
}),
], ],
}); });
} }

View File

@ -15,7 +15,7 @@ function scrapeSceneMetadata(data, channel) {
release.description = data.description; release.description = data.description;
release.date = new Date(data.release_date); release.date = new Date(data.release_date);
release.duration = qu.durationToSeconds(data.videos_duration); release.duration = data.seconds_duration || qu.durationToSeconds(data.videos_duration);
release.actors = data.models.map((model) => ({ release.actors = data.models.map((model) => ({
entryId: model.id, entryId: model.id,
@ -50,6 +50,22 @@ function scrapeAllMetadata(scenes, channel) {
return scenes.map((data) => scrapeSceneMetadata(data, channel)); return scenes.map((data) => scrapeSceneMetadata(data, channel));
} }
function scrapeAllApi(scenes, channel) {
return scenes.map((data) => {
const release = {};
release.entryId = data.id;
release.title = data.title;
release.description = data.description;
console.log(data);
console.log(release);
return release;
});
}
function scrapeProfileMetadata(data, channel) { function scrapeProfileMetadata(data, channel) {
const profile = {}; const profile = {};
@ -95,6 +111,16 @@ async function fetchLatestMetadata(channel, page = 1) {
return res.status; return res.status;
} }
async function fetchLatestApi(channel, page, { parameters }) {
const res = await http.get(`${channel.url}/_next/data/${parameters.endpoint}/videos.json?order_by=publish_date&sort_by=desc&per_page=30&page=${page}`);
if (res.ok) {
return scrapeAllApi(res.body.pageProps.contents.data, channel);
}
return res.status;
}
async function fetchSceneMetadata(url, channel) { async function fetchSceneMetadata(url, channel) {
const res = await http.get(url, { const res = await http.get(url, {
parse: true, parse: true,
@ -136,8 +162,12 @@ async function fetchProfileMetadata(actor, channel) {
module.exports = { module.exports = {
metadata: { metadata: {
// probably deprecated
fetchLatest: fetchLatestMetadata, fetchLatest: fetchLatestMetadata,
fetchScene: fetchSceneMetadata, fetchScene: fetchSceneMetadata,
fetchProfile: fetchProfileMetadata, fetchProfile: fetchProfileMetadata,
}, },
api: {
fetchLatest: fetchLatestApi,
},
}; };

View File

@ -0,0 +1,12 @@
'use strict';
const { updateSceneSearch, updateMovieSearch } = require('../update-search');
async function init() {
await updateSceneSearch();
await updateMovieSearch();
process.exit();
}
init();