2 Commits

Author SHA1 Message Date
boi12321
4c9cbe11aa refactor gender parsing function 2023-05-27 17:49:19 +02:00
boi12321
6b92b2020d refactor age check 2023-05-27 17:46:31 +02:00
22 changed files with 1870 additions and 352 deletions

2043
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@@ -115,6 +115,7 @@
"longjohn": "^0.2.12", "longjohn": "^0.2.12",
"mime": "^2.4.4", "mime": "^2.4.4",
"mitt": "^3.0.0", "mitt": "^3.0.0",
"moment": "^2.24.0",
"nanoid": "^3.1.30", "nanoid": "^3.1.30",
"node-fetch": "^2.6.7", "node-fetch": "^2.6.7",
"object-merge-advanced": "^12.1.0", "object-merge-advanced": "^12.1.0",

View File

@@ -3,7 +3,7 @@
const config = require('config'); const config = require('config');
const util = require('util'); const util = require('util');
const Promise = require('bluebird'); const Promise = require('bluebird');
const dayjs = require('dayjs'); const moment = require('moment');
const blake2 = require('blake2'); const blake2 = require('blake2');
const DOMPurify = require('dompurify'); const DOMPurify = require('dompurify');
const { JSDOM } = require('jsdom'); const { JSDOM } = require('jsdom');
@@ -33,6 +33,7 @@ const capitalize = require('./utils/capitalize');
const resolvePlace = require('./utils/resolve-place'); const resolvePlace = require('./utils/resolve-place');
const { resolveLayoutScraper } = require('./scrapers/resolve'); const { resolveLayoutScraper } = require('./scrapers/resolve');
const getRecursiveParameters = require('./utils/get-recursive-parameters'); const getRecursiveParameters = require('./utils/get-recursive-parameters');
const dayjs = require('dayjs');
const hairColors = { const hairColors = {
'jet-black': 'black', 'jet-black': 'black',
@@ -133,7 +134,7 @@ function getMostFrequentDate(dates) {
return null; return null;
} }
return dayjs({ year, month, date }).toDate(); return moment({ year, month, date }).toDate();
} }
function getHighest(items) { function getHighest(items) {
@@ -334,6 +335,32 @@ function curateProfileEntry(profile) {
return curatedProfileEntry; return curatedProfileEntry;
} }
/**
* @param {Date} birthDate
*/
function isAdult(birthDate) {
return dayjs(birthDate).diff(new Date(), "years") >= 18;
}
/**
* Tries to parse gender from string, or null if not found
*
* @param {String} text
* @returns {String | null}
*/
function tryParseGender(text) {
if (/female/i.test(profile.gender)) {
return 'female';
}
else if (/shemale|trans/i.test(profile.gender)) {
return 'transsexual';
}
else if (/male/i.test(profile.gender)) {
return 'male';
}
return null;
}
async function curateProfile(profile, actor) { async function curateProfile(profile, actor) {
if (!profile) { if (!profile) {
return null; return null;
@@ -367,15 +394,12 @@ async function curateProfile(profile, actor) {
curatedProfile.tattoos = profile.tattoos?.trim() || null; curatedProfile.tattoos = profile.tattoos?.trim() || null;
curatedProfile.piercings = profile.piercings?.trim() || null; curatedProfile.piercings = profile.piercings?.trim() || null;
curatedProfile.gender = (/female/i.test(profile.gender) && 'female') curatedProfile.gender = tryParseGender(profile.gender);
|| (/shemale|trans/i.test(profile.gender) && 'transsexual')
|| (/male/i.test(profile.gender) && 'male')
|| null;
const dateOfBirth = profile.dateOfBirth || profile.birthdate; const dateOfBirth = profile.dateOfBirth || profile.birthdate;
curatedProfile.dateOfBirth = (!Number.isNaN(Number(dateOfBirth)) // possibly valid date curatedProfile.dateOfBirth = (!Number.isNaN(Number(dateOfBirth)) // possibly valid date
&& new Date() - dateOfBirth > 567648000000 // over 18 && isAdult(dateOfBirth) // over 18
&& dateOfBirth) && dateOfBirth)
|| null; || null;

View File

@@ -2,10 +2,7 @@
const config = require('config'); const config = require('config');
const yargs = require('yargs'); const yargs = require('yargs');
const dayjs = require('dayjs'); const moment = require('moment');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
function interpretAfter(after, ignoreIfEmpty = false) { function interpretAfter(after, ignoreIfEmpty = false) {
if (!after && ignoreIfEmpty) { if (!after && ignoreIfEmpty) {
@@ -18,13 +15,13 @@ function interpretAfter(after, ignoreIfEmpty = false) {
if (/\d{2,4}-\d{2}-\d{2,4}/.test(after)) { if (/\d{2,4}-\d{2}-\d{2,4}/.test(after)) {
// using date // using date
return dayjs return moment
.utc(after, ['YYYY-MM-DD', 'DD-MM-YYYY']) .utc(after, ['YYYY-MM-DD', 'DD-MM-YYYY'])
.toDate(); .toDate();
} }
// using time distance (e.g. "1 month") // using time distance (e.g. "1 month")
return dayjs return moment
.utc() .utc()
.subtract(...after.split(' ')) .subtract(...after.split(' '))
.toDate(); .toDate();

View File

@@ -2,10 +2,7 @@
const { JSDOM } = require('jsdom'); const { JSDOM } = require('jsdom');
const cheerio = require('cheerio'); const cheerio = require('cheerio');
const dayjs = require('dayjs'); const moment = require('moment');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const http = require('../utils/http'); const http = require('../utils/http');
const slugify = require('../utils/slugify'); const slugify = require('../utils/slugify');
@@ -58,7 +55,7 @@ function scrapeAll(html) {
const { shootId, title } = extractTitle(originalTitle); const { shootId, title } = extractTitle(originalTitle);
const entryId = new URL(url).pathname.split('/')[2]; const entryId = new URL(url).pathname.split('/')[2];
const date = dayjs.utc($(element).attr('release'), 'YYYY/MM/DD').toDate(); const date = moment.utc($(element).attr('release'), 'YYYY/MM/DD').toDate();
const sceneId = $(element).attr('data-content'); const sceneId = $(element).attr('data-content');
const posterElement = $(element).find('.thumbnail-avatar'); const posterElement = $(element).find('.thumbnail-avatar');
@@ -91,7 +88,7 @@ async function scrapeScene(html, url, site, useGallery) {
release.entryId = new URL(url).pathname.split('/')[2]; release.entryId = new URL(url).pathname.split('/')[2];
release.title = title; release.title = title;
release.date = dayjs.utc($('span[title="Release date"] a').text(), 'YYYY-MM-DD').toDate(); release.date = moment.utc($('span[title="Release date"] a').text(), 'YYYY-MM-DD').toDate();
const [actorsElement, tagsElement, descriptionElement] = $('.scene-description__row').toArray(); const [actorsElement, tagsElement, descriptionElement] = $('.scene-description__row').toArray();
@@ -102,7 +99,7 @@ async function scrapeScene(html, url, site, useGallery) {
.find('a[href*="com/model"]') .find('a[href*="com/model"]')
.map((actorIndex, actorElement) => $(actorElement).text()).toArray(); .map((actorIndex, actorElement) => $(actorElement).text()).toArray();
release.duration = dayjs.duration($('span[title="Runtime"]').text().trim()).asSeconds(); release.duration = moment.duration($('span[title="Runtime"]').text().trim()).asSeconds();
release.tags = $(tagsElement).find('a').map((tagIndex, tagElement) => $(tagElement).text()).toArray(); release.tags = $(tagsElement).find('a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
const photos = useGallery const photos = useGallery

View File

@@ -2,10 +2,7 @@
/* eslint-disable newline-per-chained-call */ /* eslint-disable newline-per-chained-call */
const cheerio = require('cheerio'); const cheerio = require('cheerio');
const dayjs = require('dayjs'); const moment = require('moment');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const logger = require('../logger')(__filename); const logger = require('../logger')(__filename);
const slugify = require('../utils/slugify'); const slugify = require('../utils/slugify');
@@ -30,7 +27,7 @@ function scrape(html, site) {
release.shootId = sceneLinkElement.attr('id') && sceneLinkElement.attr('id').split('-')[1]; release.shootId = sceneLinkElement.attr('id') && sceneLinkElement.attr('id').split('-')[1];
release.entryId = new URL(release.url).pathname.match(/video(\d+)/)?.[1]; release.entryId = new URL(release.url).pathname.match(/video(\d+)/)?.[1];
release.date = dayjs.utc($(element).find('.thmb_mr_2 span.faTxt').text(), 'MMM D, YYYY').toDate(); release.date = moment.utc($(element).find('.thmb_mr_2 span.faTxt').text(), 'MMM D, YYYY').toDate();
release.actors = $(element).find('.cast-wrapper a.cast').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray(); release.actors = $(element).find('.cast-wrapper a.cast').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray();
const photoElement = $(element).find('.rollover-image'); const photoElement = $(element).find('.rollover-image');
@@ -40,7 +37,7 @@ function scrape(html, site) {
release.poster = `https:${photoElement.attr('data-original')}`; release.poster = `https:${photoElement.attr('data-original')}`;
release.photos = Array.from({ length: photosMaxIndex }, (val, index) => `https:${photosUrl}big${index + 1}.jpg`); release.photos = Array.from({ length: photosMaxIndex }, (val, index) => `https:${photosUrl}big${index + 1}.jpg`);
release.duration = dayjs.duration(`0:${$(element).find('.thmb_pic b.tTm').text()}`).asSeconds(); release.duration = moment.duration(`0:${$(element).find('.thmb_pic b.tTm').text()}`).asSeconds();
release.channel = $(element).find('a[href*="/websites"]').attr('href').split('/').slice(-1)[0]; release.channel = $(element).find('a[href*="/websites"]').attr('href').split('/').slice(-1)[0];
return release; return release;

View File

@@ -1,10 +1,7 @@
'use strict'; 'use strict';
const { JSDOM } = require('jsdom'); const { JSDOM } = require('jsdom');
const dayjs = require('dayjs'); const moment = require('moment');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const http = require('../utils/http'); const http = require('../utils/http');
@@ -39,7 +36,7 @@ function scrapeProfile(html, actorName) {
return acc; return acc;
}, {}); }, {});
if (bio.dateOfBirth) profile.birthdate = dayjs.utc(bio.dateOfBirth, 'YYYY-MM-DD').toDate(); if (bio.dateOfBirth) profile.birthdate = moment.utc(bio.dateOfBirth, 'YYYY-MM-DD').toDate();
if (profile.placeOfBirth && bio.country) profile.birthPlace = `${bio.placeOfBirth}, ${bio.country}`; if (profile.placeOfBirth && bio.country) profile.birthPlace = `${bio.placeOfBirth}, ${bio.country}`;
else if (bio.country) profile.birthPlace = bio.country; else if (bio.country) profile.birthPlace = bio.country;

View File

@@ -3,11 +3,8 @@
const Promise = require('bluebird'); const Promise = require('bluebird');
const util = require('util'); const util = require('util');
const { JSDOM } = require('jsdom'); const { JSDOM } = require('jsdom');
const dayjs = require('dayjs'); const moment = require('moment');
const format = require('template-format'); const format = require('template-format');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const logger = require('../logger')(__filename); const logger = require('../logger')(__filename);
const qu = require('../utils/qu'); const qu = require('../utils/qu');
@@ -244,7 +241,7 @@ async function scrapeApiReleases(json, site) {
if (site.parameters?.scene) release.url = `${site.parameters.scene}${release.path}`; if (site.parameters?.scene) release.url = `${site.parameters.scene}${release.path}`;
else if (site.url && site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`; else if (site.url && site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`;
release.date = dayjs.utc(scene.release_date, 'YYYY-MM-DD').toDate(); release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate();
release.director = scene.directors[0]?.name || null; release.director = scene.directors[0]?.name || null;
release.actors = scene.actors.map((actor) => ({ release.actors = scene.actors.map((actor) => ({

View File

@@ -1,6 +1,6 @@
'use strict'; 'use strict';
const dayjs = require('dayjs'); const moment = require('moment');
const qu = require('../utils/q'); const qu = require('../utils/q');
const slugify = require('../utils/slugify'); const slugify = require('../utils/slugify');
@@ -202,7 +202,7 @@ async function scrapeScene({ query, html }, url, channel) {
} }
async function fetchLatest(channel, page = 1) { async function fetchLatest(channel, page = 1) {
const year = dayjs().subtract(page - 1, ' year').year(); const year = moment().subtract(page - 1, ' year').year();
const url = `${channel.url}/Collections/Date/${year}`; const url = `${channel.url}/Collections/Date/${year}`;
const res = await qu.getAll(url, '.collectionGridLayout li'); const res = await qu.getAll(url, '.collectionGridLayout li');

View File

@@ -4,10 +4,7 @@ const util = require('util');
const Promise = require('bluebird'); const Promise = require('bluebird');
const cheerio = require('cheerio'); const cheerio = require('cheerio');
const { JSDOM } = require('jsdom'); const { JSDOM } = require('jsdom');
const dayjs = require('dayjs'); const moment = require('moment');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const qu = require('../utils/qu'); const qu = require('../utils/qu');
const http = require('../utils/http'); const http = require('../utils/http');
@@ -223,7 +220,7 @@ function scrapeUpcoming(html, site) {
.trim() .trim()
.split(', '); .split(', ');
release.date = dayjs release.date = moment
.utc($(element).find('.update_date_comingsoon').text().slice(7), 'MM/DD/YYYY') .utc($(element).find('.update_date_comingsoon').text().slice(7), 'MM/DD/YYYY')
.toDate(); .toDate();

View File

@@ -4,7 +4,7 @@
const Promise = require('bluebird'); const Promise = require('bluebird');
const { CookieJar } = Promise.promisifyAll(require('tough-cookie')); const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
const cookie = require('cookie'); const cookie = require('cookie');
const dayjs = require('dayjs'); const moment = require('moment');
const qu = require('../utils/qu'); const qu = require('../utils/qu');
const slugify = require('../utils/slugify'); const slugify = require('../utils/slugify');
@@ -301,7 +301,7 @@ async function fetchLatest(site, page = 1, options) {
? options.beforeNetwork ? options.beforeNetwork
: await getSession(site, options.parameters, url); : await getSession(site, options.parameters, url);
const beforeDate = dayjs().add('1', 'day').format('YYYY-MM-DD'); const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
const limit = 24; const limit = 24;
const apiUrl = site.parameters?.native || site.parameters?.extract const apiUrl = site.parameters?.native || site.parameters?.extract
? `https://site-api.project1service.com/v2/releases?dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene` ? `https://site-api.project1service.com/v2/releases?dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`

View File

@@ -2,10 +2,7 @@
/* eslint-disable newline-per-chained-call */ /* eslint-disable newline-per-chained-call */
const cheerio = require('cheerio'); const cheerio = require('cheerio');
const dayjs = require('dayjs'); const moment = require('moment');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const http = require('../utils/http'); const http = require('../utils/http');
const slugify = require('../utils/slugify'); const slugify = require('../utils/slugify');
@@ -32,7 +29,7 @@ function scrapeLatest(html, site) {
const url = `${protocol}//${hostname}${pathname}`; const url = `${protocol}//${hostname}${pathname}`;
const { title, entryId } = titleExtractor(pathname); const { title, entryId } = titleExtractor(pathname);
const date = dayjs.utc(element.find('.entry-date').text(), 'MMM D, YYYY').toDate(); const date = moment.utc(element.find('.entry-date').text(), 'MMM D, YYYY').toDate();
const actors = element.find('.contain-actors a').map((actorIndex, actorElement) => $(actorElement).text()).toArray(); const actors = element.find('.contain-actors a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
const duration = Number(element.find('.scene-runtime').text().slice(0, -4)) * 60; const duration = Number(element.find('.scene-runtime').text().slice(0, -4)) * 60;
@@ -65,7 +62,7 @@ function scrapeScene(html, url, site) {
const title = sceneElement.find('h1.scene-title').text(); const title = sceneElement.find('h1.scene-title').text();
const description = sceneElement.find('.synopsis').contents().slice(2).text().replace(/[\s\n]+/g, ' ').trim(); const description = sceneElement.find('.synopsis').contents().slice(2).text().replace(/[\s\n]+/g, ' ').trim();
const date = dayjs.utc(sceneElement.find('span.entry-date').text()?.match(/\w+ \d{1,2}, \d{4}/), 'MMM D, YYYY').toDate(); const date = moment.utc(sceneElement.find('span.entry-date').text()?.match(/\w+ \d{1,2}, \d{4}/), 'MMM D, YYYY').toDate();
const actors = $('.performer-list a, h1 a.scene-title').map((actorIndex, actorElement) => $(actorElement).text()).toArray(); const actors = $('.performer-list a, h1 a.scene-title').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
const duration = Number(sceneElement.find('.duration-ratings .duration').text().slice(10, -4)) * 60; const duration = Number(sceneElement.find('.duration-ratings .duration').text().slice(10, -4)) * 60;

View File

@@ -1,10 +1,7 @@
'use strict'; 'use strict';
const { JSDOM } = require('jsdom'); const { JSDOM } = require('jsdom');
const dayjs = require('dayjs'); const moment = require('moment');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const http = require('../utils/http'); const http = require('../utils/http');
@@ -34,8 +31,8 @@ async function scrapeProfile(html, _url, actorName) {
if (descriptionString) profile.description = descriptionString.textContent; if (descriptionString) profile.description = descriptionString.textContent;
if (bio.Birthday && !/-0001/.test(bio.Birthday)) profile.birthdate = dayjs.utc(bio.Birthday, 'MMM D, YYYY').toDate(); // birthyear sometimes -0001, see Spencer Bradley as of january 2020 if (bio.Birthday && !/-0001/.test(bio.Birthday)) profile.birthdate = moment.utc(bio.Birthday, 'MMM D, YYYY').toDate(); // birthyear sometimes -0001, see Spencer Bradley as of january 2020
if (bio.Born) profile.birthdate = dayjs.utc(bio.Born, 'YYYY-MM-DD').toDate(); if (bio.Born) profile.birthdate = moment.utc(bio.Born, 'YYYY-MM-DD').toDate();
profile.birthPlace = bio['Birth Place'] || bio.Birthplace; profile.birthPlace = bio['Birth Place'] || bio.Birthplace;
profile.residencePlace = bio['City and Country']; profile.residencePlace = bio['City and Country'];

View File

@@ -2,10 +2,7 @@
/* eslint-disable newline-per-chained-call */ /* eslint-disable newline-per-chained-call */
const cheerio = require('cheerio'); const cheerio = require('cheerio');
const dayjs = require('dayjs'); const moment = require('moment');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const { get, geta } = require('../utils/q'); const { get, geta } = require('../utils/q');
const slugify = require('../utils/slugify'); const slugify = require('../utils/slugify');
@@ -38,7 +35,7 @@ function scrapeLatest(html, site) {
const titleText = thumbnailElement.attr('alt'); const titleText = thumbnailElement.attr('alt');
const title = titleText.slice(titleText.indexOf(':') + 1).trim(); const title = titleText.slice(titleText.indexOf(':') + 1).trim();
const date = dayjs.utc($(element).find('.scene-date'), ['MM/DD/YYYY', 'YYYY-MM-DD']).toDate(); const date = moment.utc($(element).find('.scene-date'), ['MM/DD/YYYY', 'YYYY-MM-DD']).toDate();
const actors = $(element).find('.scene-models a').map((actorIndex, actorElement) => $(actorElement).text()).toArray(); const actors = $(element).find('.scene-models a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
const likes = Number($(element).find('.scene-votes').text()); const likes = Number($(element).find('.scene-votes').text());
@@ -73,7 +70,7 @@ async function scrapeScene(html, url, site) {
release.title = $('.video-wrapper meta[itemprop="name"]').attr('content'); release.title = $('.video-wrapper meta[itemprop="name"]').attr('content');
release.description = $('.video-wrapper meta[itemprop="description"]').attr('content'); release.description = $('.video-wrapper meta[itemprop="description"]').attr('content');
release.date = dayjs.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate(); release.date = moment.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate();
release.actors = $('.content-wrapper .scene-models-list a').map((actorIndex, actorElement) => $(actorElement).text()).toArray(); release.actors = $('.content-wrapper .scene-models-list a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
const timestamp = $('.video-wrapper meta[itemprop="duration"]').attr('content'); const timestamp = $('.video-wrapper meta[itemprop="duration"]').attr('content');

View File

@@ -1,9 +1,6 @@
'use strict'; 'use strict';
const dayjs = require('dayjs'); const moment = require('moment');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const logger = require('../logger')(__filename); const logger = require('../logger')(__filename);
const http = require('../utils/http'); const http = require('../utils/http');
@@ -25,7 +22,7 @@ function scrapeAll(scenes, entity) {
} }
release.description = scene.description; release.description = scene.description;
release.date = dayjs.utc(scene.year, 'YYYY').toDate(); release.date = moment.utc(scene.year, 'YYYY').toDate();
release.datePrecision = 'year'; release.datePrecision = 'year';
release.actors = scene.actors.map((actor) => ({ release.actors = scene.actors.map((actor) => ({

View File

@@ -5,7 +5,7 @@
const config = require('config'); const config = require('config');
const faker = require('faker'); const faker = require('faker');
const { nanoid } = require('nanoid'); const { nanoid } = require('nanoid');
const dayjs = require('dayjs'); const moment = require('moment');
const knex = require('../knex'); const knex = require('../knex');
const capitalize = require('../utils/capitalize'); const capitalize = require('../utils/capitalize');
@@ -236,7 +236,7 @@ async function fetchLatest(entity, page, options) {
const release = {}; const release = {};
release.entryId = nanoid(); release.entryId = nanoid();
release.date = dayjs().subtract(Math.floor(Math.random() * index), 'days').toDate(); release.date = moment().subtract(Math.floor(Math.random() * index), 'days').toDate();
if (options.source) { if (options.source) {
// select from configured random image source // select from configured random image source

View File

@@ -2,10 +2,7 @@
/* eslint-disable newline-per-chained-call */ /* eslint-disable newline-per-chained-call */
const Promise = require('bluebird'); const Promise = require('bluebird');
const dayjs = require('dayjs'); const moment = require('moment');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const qu = require('../utils/qu'); const qu = require('../utils/qu');
const http = require('../utils/http'); const http = require('../utils/http');
@@ -175,7 +172,7 @@ function scrapeUpcoming(scene, site) {
.map((component) => `${component.charAt(0).toUpperCase()}${component.slice(1)}`) .map((component) => `${component.charAt(0).toUpperCase()}${component.slice(1)}`)
.join(' '); .join(' ');
release.date = dayjs.utc(scene.releaseDate).toDate(); release.date = moment.utc(scene.releaseDate).toDate();
release.datePrecision = 'minute'; release.datePrecision = 'minute';
release.actors = scene.models.map((model) => model.name); release.actors = scene.models.map((model) => model.name);

View File

@@ -1,10 +1,7 @@
'use strict'; 'use strict';
const { JSDOM } = require('jsdom'); const { JSDOM } = require('jsdom');
const dayjs = require('dayjs'); const moment = require('moment');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const http = require('../utils/http'); const http = require('../utils/http');
@@ -20,7 +17,7 @@ function scrapeLatest(html, site) {
release.url = `${origin}${scene.querySelector(':scope > a').href}`; release.url = `${origin}${scene.querySelector(':scope > a').href}`;
release.entryId = scene.dataset.videoId; release.entryId = scene.dataset.videoId;
release.title = scene.querySelector('.card-title').textContent; release.title = scene.querySelector('.card-title').textContent;
release.date = dayjs.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate(); release.date = moment.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate();
release.actors = Array.from(scene.querySelectorAll('.actors a'), (el) => el.textContent); release.actors = Array.from(scene.querySelectorAll('.actors a'), (el) => el.textContent);
// slow CDN? // slow CDN?
@@ -71,7 +68,7 @@ function scrapeScene(html, site, url) {
const durationEls = Array.from(scene.querySelectorAll('#t2019-stime span')); const durationEls = Array.from(scene.querySelectorAll('#t2019-stime span'));
if (durationEls.length > 1) { if (durationEls.length > 1) {
release.date = dayjs.utc(durationEls[0].textContent, 'MMMM DD, YYYY').toDate(); release.date = moment.utc(durationEls[0].textContent, 'MMMM DD, YYYY').toDate();
release.duration = Number(durationEls[1].textContent.match(/\d+/)[0]) * 60; release.duration = Number(durationEls[1].textContent.match(/\d+/)[0]) * 60;
} else { } else {
release.duration = Number(durationEls[0].textContent.match(/\d+/)[0]) * 60; release.duration = Number(durationEls[0].textContent.match(/\d+/)[0]) * 60;

View File

@@ -2,7 +2,7 @@
const config = require('config'); const config = require('config');
const Promise = require('bluebird'); const Promise = require('bluebird');
const dayjs = require('dayjs'); const moment = require('moment');
const argv = require('./argv'); const argv = require('./argv');
const logger = require('./logger')(__filename); const logger = require('./logger')(__filename);
@@ -99,7 +99,7 @@ function needNextPage(pageReleases, accReleases, isUpcoming, unextracted = []) {
.sort((releaseA, releaseB) => releaseB.date - releaseA.date) .sort((releaseA, releaseB) => releaseB.date - releaseA.date)
.slice(-1)[0]; .slice(-1)[0];
if (dayjs(oldestReleaseOnPage.date).isAfter(argv.after)) { if (moment(oldestReleaseOnPage.date).isAfter(argv.after)) {
// oldest release on page is newer than the specified date cut-off // oldest release on page is newer than the specified date cut-off
return true; return true;
} }
@@ -149,7 +149,7 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
const hasDates = releases.every((release) => !!release.date); const hasDates = releases.every((release) => !!release.date);
const limitedReleases = (argv.last && releases.slice(0, Math.max(argv.last, 0))) const limitedReleases = (argv.last && releases.slice(0, Math.max(argv.last, 0)))
|| (hasDates && releases.filter((release) => dayjs(release.date).isAfter(argv.after))) || (hasDates && releases.filter((release) => moment(release.date).isAfter(argv.after)))
|| releases.slice(0, Math.max(argv.missingDateLimit, 0)); || releases.slice(0, Math.max(argv.missingDateLimit, 0));
const { uniqueReleases, duplicateReleases } = argv.force const { uniqueReleases, duplicateReleases } = argv.force

View File

@@ -3,10 +3,7 @@
const config = require('config'); const config = require('config');
const path = require('path'); const path = require('path');
const fs = require('fs-extra'); const fs = require('fs-extra');
const dayjs = require('dayjs'); const moment = require('moment');
const utc = require('dayjs/plugin/utc');
dayjs.extend(utc);
const argv = require('../argv'); const argv = require('../argv');
const knex = require('../knex'); const knex = require('../knex');
@@ -28,8 +25,7 @@ async function actorPosters(actorNames) {
const source = path.join(config.media.path, poster.path); const source = path.join(config.media.path, poster.path);
const directory = path.join(config.media.path, 'extracted', poster.actor_name); const directory = path.join(config.media.path, 'extracted', poster.actor_name);
const formattedDate = dayjs.utc(poster.date).format('YYYY-MM-DD'); const target = path.join(directory, `${poster.actor_name} - ${poster.network_name}: ${poster.site_name} - ${poster.title.replace(/[/.]/g, '_') || poster.actor_name} (${moment.utc(poster.date).format('YYYY-MM-DD')})-${poster.index}.jpeg`);
const target = path.join(directory, `${poster.actor_name} - ${poster.network_name}: ${poster.site_name} - ${poster.title.replace(/[/.]/g, '_') || poster.actor_name} (${formattedDate})-${poster.index}.jpeg`);
await fs.mkdir(path.join(directory), { recursive: true }); await fs.mkdir(path.join(directory), { recursive: true });
const file = await fs.readFile(source); const file = await fs.readFile(source);
@@ -54,7 +50,7 @@ async function sitePosters(siteSlugs) {
const directory = path.join(config.media.path, 'extracted', poster.site_name); const directory = path.join(config.media.path, 'extracted', poster.site_name);
const source = path.join(config.media.path, poster.path); const source = path.join(config.media.path, poster.path);
const target = path.join(directory, `${poster.site_name} - ${dayjs.utc(poster.date).format('YYYY-MM-DD')} - ${poster.title.replace(/[/.]/g, '_')}.jpeg`); const target = path.join(directory, `${poster.site_name} - ${moment.utc(poster.date).format('YYYY-MM-DD')} - ${poster.title.replace(/[/.]/g, '_')}.jpeg`);
await fs.mkdir(directory, { recursive: true }); await fs.mkdir(directory, { recursive: true });
await fs.copyFile(source, target); await fs.copyFile(source, target);

View File

@@ -1,7 +1,7 @@
'use strict'; 'use strict';
const { JSDOM } = require('jsdom'); const { JSDOM } = require('jsdom');
const dayjs = require('dayjs'); const moment = require('moment');
const http = require('./http'); const http = require('./http');
const virtualConsole = require('./virtual-console')(__filename); const virtualConsole = require('./virtual-console')(__filename);
@@ -38,10 +38,10 @@ function extractDate(dateString, format, match) {
function formatDate(dateValue, format, inputFormat) { function formatDate(dateValue, format, inputFormat) {
if (inputFormat) { if (inputFormat) {
return dayjs(dateValue, inputFormat).format(format); return moment(dateValue, inputFormat).format(format);
} }
return dayjs(dateValue).format(format); return moment(dateValue).format(format);
} }
function durationToSeconds(durationString, match) { function durationToSeconds(durationString, match) {

View File

@@ -1,7 +1,7 @@
'use strict'; 'use strict';
const { makeExtendSchemaPlugin, gql } = require('graphile-utils'); const { makeExtendSchemaPlugin, gql } = require('graphile-utils');
const dayjs = require('dayjs'); const moment = require('moment');
const { cmToFeetInches, cmToInches, kgToLbs } = require('../../utils/convert'); const { cmToFeetInches, cmToInches, kgToLbs } = require('../../utils/convert');
const schemaExtender = makeExtendSchemaPlugin((_build) => ({ const schemaExtender = makeExtendSchemaPlugin((_build) => ({
@@ -45,12 +45,12 @@ const schemaExtender = makeExtendSchemaPlugin((_build) => ({
ageFromBirth(parent, _args, _context, _info) { ageFromBirth(parent, _args, _context, _info) {
if (!parent.dateOfBirth) return null; if (!parent.dateOfBirth) return null;
return dayjs().diff(parent.dateOfBirth, 'years'); return moment().diff(parent.dateOfBirth, 'years');
}, },
ageAtDeath(parent, _args, _context, _info) { ageAtDeath(parent, _args, _context, _info) {
if (!parent.dateOfDeath) return null; if (!parent.dateOfDeath) return null;
return dayjs(parent.dateOfDeath).diff(parent.dateOfBirth, 'years'); return moment(parent.dateOfDeath).diff(parent.dateOfBirth, 'years');
}, },
height(parent, args, _context, _info) { height(parent, args, _context, _info) {
if (!parent.height) return null; if (!parent.height) return null;