boiii
/
traxxx
forked from DebaucheryLibrarian/traxxx
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

replaced moment with dayjs

This commit is contained in:
boi12321 2023-05-26 23:10:35 +02:00
parent 811c8e1c92
commit 091b7bd119
22 changed files with 347 additions and 1841 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2043
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

1
package.json
View File

@ -115,7 +115,6 @@
"longjohn": "^0.2.12",
"mime": "^2.4.4",
"mitt": "^3.0.0",
"moment": "^2.24.0",
"nanoid": "^3.1.30",
"node-fetch": "^2.6.7",
"object-merge-advanced": "^12.1.0",

4
src/actors.js
View File

@ -3,7 +3,7 @@
const config = require('config');
const util = require('util');
const Promise = require('bluebird');
const moment = require('moment');
const dayjs = require('dayjs');
const blake2 = require('blake2');
const DOMPurify = require('dompurify');
const { JSDOM } = require('jsdom');
@ -133,7 +133,7 @@ function getMostFrequentDate(dates) {
return null;
}
return moment({ year, month, date }).toDate();
return dayjs({ year, month, date }).toDate();
}
function getHighest(items) {

9
src/argv.js
View File

@ -2,7 +2,10 @@
const config = require('config');
const yargs = require('yargs');
const moment = require('moment');
const dayjs = require('dayjs');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
function interpretAfter(after, ignoreIfEmpty = false) {
if (!after && ignoreIfEmpty) {
@ -15,13 +18,13 @@ function interpretAfter(after, ignoreIfEmpty = false) {
if (/\d{2,4}-\d{2}-\d{2,4}/.test(after)) {
// using date
return moment
return dayjs
.utc(after, ['YYYY-MM-DD', 'DD-MM-YYYY'])
.toDate();
}
// using time distance (e.g. "1 month")
return moment
return dayjs
.utc()
.subtract(...after.split(' '))
.toDate();

11
src/scrapers/analvids.js
View File

@ -2,7 +2,10 @@
const { JSDOM } = require('jsdom');
const cheerio = require('cheerio');
const moment = require('moment');
const dayjs = require('dayjs');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const http = require('../utils/http');
const slugify = require('../utils/slugify');
@ -55,7 +58,7 @@ function scrapeAll(html) {
const { shootId, title } = extractTitle(originalTitle);
const entryId = new URL(url).pathname.split('/')[2];
const date = moment.utc($(element).attr('release'), 'YYYY/MM/DD').toDate();
const date = dayjs.utc($(element).attr('release'), 'YYYY/MM/DD').toDate();
const sceneId = $(element).attr('data-content');
const posterElement = $(element).find('.thumbnail-avatar');
@ -88,7 +91,7 @@ async function scrapeScene(html, url, site, useGallery) {
release.entryId = new URL(url).pathname.split('/')[2];
release.title = title;
release.date = moment.utc($('span[title="Release date"] a').text(), 'YYYY-MM-DD').toDate();
release.date = dayjs.utc($('span[title="Release date"] a').text(), 'YYYY-MM-DD').toDate();
const [actorsElement, tagsElement, descriptionElement] = $('.scene-description__row').toArray();
@ -99,7 +102,7 @@ async function scrapeScene(html, url, site, useGallery) {
.find('a[href*="com/model"]')
.map((actorIndex, actorElement) => $(actorElement).text()).toArray();
release.duration = moment.duration($('span[title="Runtime"]').text().trim()).asSeconds();
release.duration = dayjs.duration($('span[title="Runtime"]').text().trim()).asSeconds();
release.tags = $(tagsElement).find('a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const photos = useGallery

9
src/scrapers/bangbros.js
View File

@ -2,7 +2,10 @@
/* eslint-disable newline-per-chained-call */
const cheerio = require('cheerio');
const moment = require('moment');
const dayjs = require('dayjs');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const logger = require('../logger')(__filename);
const slugify = require('../utils/slugify');
@ -27,7 +30,7 @@ function scrape(html, site) {
release.shootId = sceneLinkElement.attr('id') && sceneLinkElement.attr('id').split('-')[1];
release.entryId = new URL(release.url).pathname.match(/video(\d+)/)?.[1];
release.date = moment.utc($(element).find('.thmb_mr_2 span.faTxt').text(), 'MMM D, YYYY').toDate();
release.date = dayjs.utc($(element).find('.thmb_mr_2 span.faTxt').text(), 'MMM D, YYYY').toDate();
release.actors = $(element).find('.cast-wrapper a.cast').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray();
const photoElement = $(element).find('.rollover-image');
@ -37,7 +40,7 @@ function scrape(html, site) {
release.poster = `https:${photoElement.attr('data-original')}`;
release.photos = Array.from({ length: photosMaxIndex }, (val, index) => `https:${photosUrl}big${index + 1}.jpg`);
release.duration = moment.duration(`0:${$(element).find('.thmb_pic b.tTm').text()}`).asSeconds();
release.duration = dayjs.duration(`0:${$(element).find('.thmb_pic b.tTm').text()}`).asSeconds();
release.channel = $(element).find('a[href*="/websites"]').attr('href').split('/').slice(-1)[0];
return release;

7
src/scrapers/freeones.js
View File

@ -1,7 +1,10 @@
'use strict';
const { JSDOM } = require('jsdom');
const moment = require('moment');
const dayjs = require('dayjs');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const http = require('../utils/http');
@ -36,7 +39,7 @@ function scrapeProfile(html, actorName) {
return acc;
}, {});
if (bio.dateOfBirth) profile.birthdate = moment.utc(bio.dateOfBirth, 'YYYY-MM-DD').toDate();
if (bio.dateOfBirth) profile.birthdate = dayjs.utc(bio.dateOfBirth, 'YYYY-MM-DD').toDate();
if (profile.placeOfBirth && bio.country) profile.birthPlace = `${bio.placeOfBirth}, ${bio.country}`;
else if (bio.country) profile.birthPlace = bio.country;

7
src/scrapers/gamma.js
View File

@ -3,8 +3,11 @@
const Promise = require('bluebird');
const util = require('util');
const { JSDOM } = require('jsdom');
const moment = require('moment');
const dayjs = require('dayjs');
const format = require('template-format');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const logger = require('../logger')(__filename);
const qu = require('../utils/qu');
@ -241,7 +244,7 @@ async function scrapeApiReleases(json, site) {
if (site.parameters?.scene) release.url = `${site.parameters.scene}${release.path}`;
else if (site.url && site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`;
release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate();
release.date = dayjs.utc(scene.release_date, 'YYYY-MM-DD').toDate();
release.director = scene.directors[0]?.name || null;
release.actors = scene.actors.map((actor) => ({

4
src/scrapers/inthecrack.js
View File

@ -1,6 +1,6 @@
'use strict';
const moment = require('moment');
const dayjs = require('dayjs');
const qu = require('../utils/q');
const slugify = require('../utils/slugify');
@ -202,7 +202,7 @@ async function scrapeScene({ query, html }, url, channel) {
}
async function fetchLatest(channel, page = 1) {
const year = moment().subtract(page - 1, ' year').year();
const year = dayjs().subtract(page - 1, ' year').year();
const url = `${channel.url}/Collections/Date/${year}`;
const res = await qu.getAll(url, '.collectionGridLayout li');

7
src/scrapers/julesjordan.js
View File

@ -4,7 +4,10 @@ const util = require('util');
const Promise = require('bluebird');
const cheerio = require('cheerio');
const { JSDOM } = require('jsdom');
const moment = require('moment');
const dayjs = require('dayjs');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const qu = require('../utils/qu');
const http = require('../utils/http');
@ -220,7 +223,7 @@ function scrapeUpcoming(html, site) {
.trim()
.split(', ');
release.date = moment
release.date = dayjs
.utc($(element).find('.update_date_comingsoon').text().slice(7), 'MM/DD/YYYY')
.toDate();

4
src/scrapers/mindgeek.js
View File

@ -4,7 +4,7 @@
const Promise = require('bluebird');
const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
const cookie = require('cookie');
const moment = require('moment');
const dayjs = require('dayjs');
const qu = require('../utils/qu');
const slugify = require('../utils/slugify');
@ -301,7 +301,7 @@ async function fetchLatest(site, page = 1, options) {
? options.beforeNetwork
: await getSession(site, options.parameters, url);
const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
const beforeDate = dayjs().add('1', 'day').format('YYYY-MM-DD');
const limit = 24;
const apiUrl = site.parameters?.native || site.parameters?.extract
? `https://site-api.project1service.com/v2/releases?dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`

9
src/scrapers/naughtyamerica.js
View File

@ -2,7 +2,10 @@
/* eslint-disable newline-per-chained-call */
const cheerio = require('cheerio');
const moment = require('moment');
const dayjs = require('dayjs');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const http = require('../utils/http');
const slugify = require('../utils/slugify');
@ -29,7 +32,7 @@ function scrapeLatest(html, site) {
const url = `${protocol}//${hostname}${pathname}`;
const { title, entryId } = titleExtractor(pathname);
const date = moment.utc(element.find('.entry-date').text(), 'MMM D, YYYY').toDate();
const date = dayjs.utc(element.find('.entry-date').text(), 'MMM D, YYYY').toDate();
const actors = element.find('.contain-actors a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
const duration = Number(element.find('.scene-runtime').text().slice(0, -4)) * 60;
@ -62,7 +65,7 @@ function scrapeScene(html, url, site) {
const title = sceneElement.find('h1.scene-title').text();
const description = sceneElement.find('.synopsis').contents().slice(2).text().replace(/[\s\n]+/g, ' ').trim();
const date = moment.utc(sceneElement.find('span.entry-date').text()?.match(/\w+ \d{1,2}, \d{4}/), 'MMM D, YYYY').toDate();
const date = dayjs.utc(sceneElement.find('span.entry-date').text()?.match(/\w+ \d{1,2}, \d{4}/), 'MMM D, YYYY').toDate();
const actors = $('.performer-list a, h1 a.scene-title').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
const duration = Number(sceneElement.find('.duration-ratings .duration').text().slice(10, -4)) * 60;

9
src/scrapers/pornhub.js
View File

@ -1,7 +1,10 @@
'use strict';
const { JSDOM } = require('jsdom');
const moment = require('moment');
const dayjs = require('dayjs');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const http = require('../utils/http');
@ -31,8 +34,8 @@ async function scrapeProfile(html, _url, actorName) {
if (descriptionString) profile.description = descriptionString.textContent;
if (bio.Birthday && !/-0001/.test(bio.Birthday)) profile.birthdate = moment.utc(bio.Birthday, 'MMM D, YYYY').toDate(); // birthyear sometimes -0001, see Spencer Bradley as of january 2020
if (bio.Born) profile.birthdate = moment.utc(bio.Born, 'YYYY-MM-DD').toDate();
if (bio.Birthday && !/-0001/.test(bio.Birthday)) profile.birthdate = dayjs.utc(bio.Birthday, 'MMM D, YYYY').toDate(); // birthyear sometimes -0001, see Spencer Bradley as of january 2020
if (bio.Born) profile.birthdate = dayjs.utc(bio.Born, 'YYYY-MM-DD').toDate();
profile.birthPlace = bio['Birth Place'] || bio.Birthplace;
profile.residencePlace = bio['City and Country'];

9
src/scrapers/private.js
View File

@ -2,7 +2,10 @@
/* eslint-disable newline-per-chained-call */
const cheerio = require('cheerio');
const moment = require('moment');
const dayjs = require('dayjs');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const { get, geta } = require('../utils/q');
const slugify = require('../utils/slugify');
@ -35,7 +38,7 @@ function scrapeLatest(html, site) {
const titleText = thumbnailElement.attr('alt');
const title = titleText.slice(titleText.indexOf(':') + 1).trim();
const date = moment.utc($(element).find('.scene-date'), ['MM/DD/YYYY', 'YYYY-MM-DD']).toDate();
const date = dayjs.utc($(element).find('.scene-date'), ['MM/DD/YYYY', 'YYYY-MM-DD']).toDate();
const actors = $(element).find('.scene-models a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
const likes = Number($(element).find('.scene-votes').text());
@ -70,7 +73,7 @@ async function scrapeScene(html, url, site) {
release.title = $('.video-wrapper meta[itemprop="name"]').attr('content');
release.description = $('.video-wrapper meta[itemprop="description"]').attr('content');
release.date = moment.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate();
release.date = dayjs.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate();
release.actors = $('.content-wrapper .scene-models-list a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
const timestamp = $('.video-wrapper meta[itemprop="duration"]').attr('content');

7
src/scrapers/teencoreclub.js
View File

@ -1,6 +1,9 @@
'use strict';
const moment = require('moment');
const dayjs = require('dayjs');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const logger = require('../logger')(__filename);
const http = require('../utils/http');
@ -22,7 +25,7 @@ function scrapeAll(scenes, entity) {
}
release.description = scene.description;
release.date = moment.utc(scene.year, 'YYYY').toDate();
release.date = dayjs.utc(scene.year, 'YYYY').toDate();
release.datePrecision = 'year';
release.actors = scene.actors.map((actor) => ({

4
src/scrapers/traxxx.js
View File

@ -5,7 +5,7 @@
const config = require('config');
const faker = require('faker');
const { nanoid } = require('nanoid');
const moment = require('moment');
const dayjs = require('dayjs');
const knex = require('../knex');
const capitalize = require('../utils/capitalize');
@ -236,7 +236,7 @@ async function fetchLatest(entity, page, options) {
const release = {};
release.entryId = nanoid();
release.date = moment().subtract(Math.floor(Math.random() * index), 'days').toDate();
release.date = dayjs().subtract(Math.floor(Math.random() * index), 'days').toDate();
if (options.source) {
// select from configured random image source

7
src/scrapers/vixen.js
View File

@ -2,7 +2,10 @@
/* eslint-disable newline-per-chained-call */
const Promise = require('bluebird');
const moment = require('moment');
const dayjs = require('dayjs');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const qu = require('../utils/qu');
const http = require('../utils/http');
@ -172,7 +175,7 @@ function scrapeUpcoming(scene, site) {
.map((component) => `${component.charAt(0).toUpperCase()}${component.slice(1)}`)
.join(' ');
release.date = moment.utc(scene.releaseDate).toDate();
release.date = dayjs.utc(scene.releaseDate).toDate();
release.datePrecision = 'minute';
release.actors = scene.models.map((model) => model.name);

9
src/scrapers/whalemember.js
View File

@ -1,7 +1,10 @@
'use strict';
const { JSDOM } = require('jsdom');
const moment = require('moment');
const dayjs = require('dayjs');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const http = require('../utils/http');
@ -17,7 +20,7 @@ function scrapeLatest(html, site) {
release.url = `${origin}${scene.querySelector(':scope > a').href}`;
release.entryId = scene.dataset.videoId;
release.title = scene.querySelector('.card-title').textContent;
release.date = moment.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate();
release.date = dayjs.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate();
release.actors = Array.from(scene.querySelectorAll('.actors a'), (el) => el.textContent);
// slow CDN?
@ -68,7 +71,7 @@ function scrapeScene(html, site, url) {
const durationEls = Array.from(scene.querySelectorAll('#t2019-stime span'));
if (durationEls.length > 1) {
release.date = moment.utc(durationEls[0].textContent, 'MMMM DD, YYYY').toDate();
release.date = dayjs.utc(durationEls[0].textContent, 'MMMM DD, YYYY').toDate();
release.duration = Number(durationEls[1].textContent.match(/\d+/)[0]) * 60;
} else {
release.duration = Number(durationEls[0].textContent.match(/\d+/)[0]) * 60;

6
src/updates.js
View File

@ -2,7 +2,7 @@
const config = require('config');
const Promise = require('bluebird');
const moment = require('moment');
const dayjs = require('dayjs');
const argv = require('./argv');
const logger = require('./logger')(__filename);
@ -99,7 +99,7 @@ function needNextPage(pageReleases, accReleases, isUpcoming, unextracted = []) {
.sort((releaseA, releaseB) => releaseB.date - releaseA.date)
.slice(-1)[0];
if (moment(oldestReleaseOnPage.date).isAfter(argv.after)) {
if (dayjs(oldestReleaseOnPage.date).isAfter(argv.after)) {
// oldest release on page is newer than the specified date cut-off
return true;
}
@ -149,7 +149,7 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
const hasDates = releases.every((release) => !!release.date);
const limitedReleases = (argv.last && releases.slice(0, Math.max(argv.last, 0)))
|| (hasDates && releases.filter((release) => moment(release.date).isAfter(argv.after)))
|| (hasDates && releases.filter((release) => dayjs(release.date).isAfter(argv.after)))
|| releases.slice(0, Math.max(argv.missingDateLimit, 0));
const { uniqueReleases, duplicateReleases } = argv.force

10
src/utils/posters.js
View File

@ -3,7 +3,10 @@
const config = require('config');
const path = require('path');
const fs = require('fs-extra');
const moment = require('moment');
const dayjs = require('dayjs');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const argv = require('../argv');
const knex = require('../knex');
@ -25,7 +28,8 @@ async function actorPosters(actorNames) {
const source = path.join(config.media.path, poster.path);
const directory = path.join(config.media.path, 'extracted', poster.actor_name);
const target = path.join(directory, `${poster.actor_name} - ${poster.network_name}: ${poster.site_name} - ${poster.title.replace(/[/.]/g, '_') || poster.actor_name} (${moment.utc(poster.date).format('YYYY-MM-DD')})-${poster.index}.jpeg`);
const formattedDate = dayjs.utc(poster.date).format('YYYY-MM-DD');
const target = path.join(directory, `${poster.actor_name} - ${poster.network_name}: ${poster.site_name} - ${poster.title.replace(/[/.]/g, '_') || poster.actor_name} (${formattedDate})-${poster.index}.jpeg`);
await fs.mkdir(path.join(directory), { recursive: true });
const file = await fs.readFile(source);
@ -50,7 +54,7 @@ async function sitePosters(siteSlugs) {
const directory = path.join(config.media.path, 'extracted', poster.site_name);
const source = path.join(config.media.path, poster.path);
const target = path.join(directory, `${poster.site_name} - ${moment.utc(poster.date).format('YYYY-MM-DD')} - ${poster.title.replace(/[/.]/g, '_')}.jpeg`);
const target = path.join(directory, `${poster.site_name} - ${dayjs.utc(poster.date).format('YYYY-MM-DD')} - ${poster.title.replace(/[/.]/g, '_')}.jpeg`);
await fs.mkdir(directory, { recursive: true });
await fs.copyFile(source, target);

6
src/utils/qu.js
View File

@ -1,7 +1,7 @@
'use strict';
const { JSDOM } = require('jsdom');
const moment = require('moment');
const dayjs = require('dayjs');
const http = require('./http');
const virtualConsole = require('./virtual-console')(__filename);
@ -38,10 +38,10 @@ function extractDate(dateString, format, match) {
function formatDate(dateValue, format, inputFormat) {
if (inputFormat) {
return moment(dateValue, inputFormat).format(format);
return dayjs(dateValue, inputFormat).format(format);
}
return moment(dateValue).format(format);
return dayjs(dateValue).format(format);
}
function durationToSeconds(durationString, match) {

6
src/web/plugins/actors.js
View File

@ -1,7 +1,7 @@
'use strict';
const { makeExtendSchemaPlugin, gql } = require('graphile-utils');
const moment = require('moment');
const dayjs = require('dayjs');
const { cmToFeetInches, cmToInches, kgToLbs } = require('../../utils/convert');
const schemaExtender = makeExtendSchemaPlugin((_build) => ({
@ -45,12 +45,12 @@ const schemaExtender = makeExtendSchemaPlugin((_build) => ({
ageFromBirth(parent, _args, _context, _info) {
if (!parent.dateOfBirth) return null;
return moment().diff(parent.dateOfBirth, 'years');
return dayjs().diff(parent.dateOfBirth, 'years');
},
ageAtDeath(parent, _args, _context, _info) {
if (!parent.dateOfDeath) return null;
return moment(parent.dateOfDeath).diff(parent.dateOfBirth, 'years');
return dayjs(parent.dateOfDeath).diff(parent.dateOfBirth, 'years');
},
height(parent, args, _context, _info) {
if (!parent.height) return null;