forked from DebaucheryLibrarian/traxxx
Compare commits
1 Commits
master
...
091b7bd119
Author | SHA1 | Date |
---|---|---|
boi12321 | 091b7bd119 |
File diff suppressed because it is too large
Load Diff
|
@ -115,7 +115,6 @@
|
||||||
"longjohn": "^0.2.12",
|
"longjohn": "^0.2.12",
|
||||||
"mime": "^2.4.4",
|
"mime": "^2.4.4",
|
||||||
"mitt": "^3.0.0",
|
"mitt": "^3.0.0",
|
||||||
"moment": "^2.24.0",
|
|
||||||
"nanoid": "^3.1.30",
|
"nanoid": "^3.1.30",
|
||||||
"node-fetch": "^2.6.7",
|
"node-fetch": "^2.6.7",
|
||||||
"object-merge-advanced": "^12.1.0",
|
"object-merge-advanced": "^12.1.0",
|
||||||
|
|
|
@ -3,7 +3,7 @@
|
||||||
const config = require('config');
|
const config = require('config');
|
||||||
const util = require('util');
|
const util = require('util');
|
||||||
const Promise = require('bluebird');
|
const Promise = require('bluebird');
|
||||||
const moment = require('moment');
|
const dayjs = require('dayjs');
|
||||||
const blake2 = require('blake2');
|
const blake2 = require('blake2');
|
||||||
const DOMPurify = require('dompurify');
|
const DOMPurify = require('dompurify');
|
||||||
const { JSDOM } = require('jsdom');
|
const { JSDOM } = require('jsdom');
|
||||||
|
@ -133,7 +133,7 @@ function getMostFrequentDate(dates) {
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
return moment({ year, month, date }).toDate();
|
return dayjs({ year, month, date }).toDate();
|
||||||
}
|
}
|
||||||
|
|
||||||
function getHighest(items) {
|
function getHighest(items) {
|
||||||
|
|
|
@ -2,7 +2,10 @@
|
||||||
|
|
||||||
const config = require('config');
|
const config = require('config');
|
||||||
const yargs = require('yargs');
|
const yargs = require('yargs');
|
||||||
const moment = require('moment');
|
const dayjs = require('dayjs');
|
||||||
|
const utc = require('dayjs/plugin/utc');
|
||||||
|
|
||||||
|
dayjs.extend(utc);
|
||||||
|
|
||||||
function interpretAfter(after, ignoreIfEmpty = false) {
|
function interpretAfter(after, ignoreIfEmpty = false) {
|
||||||
if (!after && ignoreIfEmpty) {
|
if (!after && ignoreIfEmpty) {
|
||||||
|
@ -15,13 +18,13 @@ function interpretAfter(after, ignoreIfEmpty = false) {
|
||||||
|
|
||||||
if (/\d{2,4}-\d{2}-\d{2,4}/.test(after)) {
|
if (/\d{2,4}-\d{2}-\d{2,4}/.test(after)) {
|
||||||
// using date
|
// using date
|
||||||
return moment
|
return dayjs
|
||||||
.utc(after, ['YYYY-MM-DD', 'DD-MM-YYYY'])
|
.utc(after, ['YYYY-MM-DD', 'DD-MM-YYYY'])
|
||||||
.toDate();
|
.toDate();
|
||||||
}
|
}
|
||||||
|
|
||||||
// using time distance (e.g. "1 month")
|
// using time distance (e.g. "1 month")
|
||||||
return moment
|
return dayjs
|
||||||
.utc()
|
.utc()
|
||||||
.subtract(...after.split(' '))
|
.subtract(...after.split(' '))
|
||||||
.toDate();
|
.toDate();
|
||||||
|
|
|
@ -2,7 +2,10 @@
|
||||||
|
|
||||||
const { JSDOM } = require('jsdom');
|
const { JSDOM } = require('jsdom');
|
||||||
const cheerio = require('cheerio');
|
const cheerio = require('cheerio');
|
||||||
const moment = require('moment');
|
const dayjs = require('dayjs');
|
||||||
|
const utc = require('dayjs/plugin/utc');
|
||||||
|
|
||||||
|
dayjs.extend(utc);
|
||||||
|
|
||||||
const http = require('../utils/http');
|
const http = require('../utils/http');
|
||||||
const slugify = require('../utils/slugify');
|
const slugify = require('../utils/slugify');
|
||||||
|
@ -55,7 +58,7 @@ function scrapeAll(html) {
|
||||||
const { shootId, title } = extractTitle(originalTitle);
|
const { shootId, title } = extractTitle(originalTitle);
|
||||||
const entryId = new URL(url).pathname.split('/')[2];
|
const entryId = new URL(url).pathname.split('/')[2];
|
||||||
|
|
||||||
const date = moment.utc($(element).attr('release'), 'YYYY/MM/DD').toDate();
|
const date = dayjs.utc($(element).attr('release'), 'YYYY/MM/DD').toDate();
|
||||||
|
|
||||||
const sceneId = $(element).attr('data-content');
|
const sceneId = $(element).attr('data-content');
|
||||||
const posterElement = $(element).find('.thumbnail-avatar');
|
const posterElement = $(element).find('.thumbnail-avatar');
|
||||||
|
@ -88,7 +91,7 @@ async function scrapeScene(html, url, site, useGallery) {
|
||||||
release.entryId = new URL(url).pathname.split('/')[2];
|
release.entryId = new URL(url).pathname.split('/')[2];
|
||||||
|
|
||||||
release.title = title;
|
release.title = title;
|
||||||
release.date = moment.utc($('span[title="Release date"] a').text(), 'YYYY-MM-DD').toDate();
|
release.date = dayjs.utc($('span[title="Release date"] a').text(), 'YYYY-MM-DD').toDate();
|
||||||
|
|
||||||
const [actorsElement, tagsElement, descriptionElement] = $('.scene-description__row').toArray();
|
const [actorsElement, tagsElement, descriptionElement] = $('.scene-description__row').toArray();
|
||||||
|
|
||||||
|
@ -99,7 +102,7 @@ async function scrapeScene(html, url, site, useGallery) {
|
||||||
.find('a[href*="com/model"]')
|
.find('a[href*="com/model"]')
|
||||||
.map((actorIndex, actorElement) => $(actorElement).text()).toArray();
|
.map((actorIndex, actorElement) => $(actorElement).text()).toArray();
|
||||||
|
|
||||||
release.duration = moment.duration($('span[title="Runtime"]').text().trim()).asSeconds();
|
release.duration = dayjs.duration($('span[title="Runtime"]').text().trim()).asSeconds();
|
||||||
release.tags = $(tagsElement).find('a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
|
release.tags = $(tagsElement).find('a').map((tagIndex, tagElement) => $(tagElement).text()).toArray();
|
||||||
|
|
||||||
const photos = useGallery
|
const photos = useGallery
|
||||||
|
|
|
@ -2,7 +2,10 @@
|
||||||
|
|
||||||
/* eslint-disable newline-per-chained-call */
|
/* eslint-disable newline-per-chained-call */
|
||||||
const cheerio = require('cheerio');
|
const cheerio = require('cheerio');
|
||||||
const moment = require('moment');
|
const dayjs = require('dayjs');
|
||||||
|
const utc = require('dayjs/plugin/utc');
|
||||||
|
|
||||||
|
dayjs.extend(utc);
|
||||||
|
|
||||||
const logger = require('../logger')(__filename);
|
const logger = require('../logger')(__filename);
|
||||||
const slugify = require('../utils/slugify');
|
const slugify = require('../utils/slugify');
|
||||||
|
@ -27,7 +30,7 @@ function scrape(html, site) {
|
||||||
release.shootId = sceneLinkElement.attr('id') && sceneLinkElement.attr('id').split('-')[1];
|
release.shootId = sceneLinkElement.attr('id') && sceneLinkElement.attr('id').split('-')[1];
|
||||||
release.entryId = new URL(release.url).pathname.match(/video(\d+)/)?.[1];
|
release.entryId = new URL(release.url).pathname.match(/video(\d+)/)?.[1];
|
||||||
|
|
||||||
release.date = moment.utc($(element).find('.thmb_mr_2 span.faTxt').text(), 'MMM D, YYYY').toDate();
|
release.date = dayjs.utc($(element).find('.thmb_mr_2 span.faTxt').text(), 'MMM D, YYYY').toDate();
|
||||||
release.actors = $(element).find('.cast-wrapper a.cast').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray();
|
release.actors = $(element).find('.cast-wrapper a.cast').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray();
|
||||||
|
|
||||||
const photoElement = $(element).find('.rollover-image');
|
const photoElement = $(element).find('.rollover-image');
|
||||||
|
@ -37,7 +40,7 @@ function scrape(html, site) {
|
||||||
release.poster = `https:${photoElement.attr('data-original')}`;
|
release.poster = `https:${photoElement.attr('data-original')}`;
|
||||||
release.photos = Array.from({ length: photosMaxIndex }, (val, index) => `https:${photosUrl}big${index + 1}.jpg`);
|
release.photos = Array.from({ length: photosMaxIndex }, (val, index) => `https:${photosUrl}big${index + 1}.jpg`);
|
||||||
|
|
||||||
release.duration = moment.duration(`0:${$(element).find('.thmb_pic b.tTm').text()}`).asSeconds();
|
release.duration = dayjs.duration(`0:${$(element).find('.thmb_pic b.tTm').text()}`).asSeconds();
|
||||||
release.channel = $(element).find('a[href*="/websites"]').attr('href').split('/').slice(-1)[0];
|
release.channel = $(element).find('a[href*="/websites"]').attr('href').split('/').slice(-1)[0];
|
||||||
|
|
||||||
return release;
|
return release;
|
||||||
|
|
|
@ -1,7 +1,10 @@
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const { JSDOM } = require('jsdom');
|
const { JSDOM } = require('jsdom');
|
||||||
const moment = require('moment');
|
const dayjs = require('dayjs');
|
||||||
|
const utc = require('dayjs/plugin/utc');
|
||||||
|
|
||||||
|
dayjs.extend(utc);
|
||||||
|
|
||||||
const http = require('../utils/http');
|
const http = require('../utils/http');
|
||||||
|
|
||||||
|
@ -36,7 +39,7 @@ function scrapeProfile(html, actorName) {
|
||||||
return acc;
|
return acc;
|
||||||
}, {});
|
}, {});
|
||||||
|
|
||||||
if (bio.dateOfBirth) profile.birthdate = moment.utc(bio.dateOfBirth, 'YYYY-MM-DD').toDate();
|
if (bio.dateOfBirth) profile.birthdate = dayjs.utc(bio.dateOfBirth, 'YYYY-MM-DD').toDate();
|
||||||
|
|
||||||
if (profile.placeOfBirth && bio.country) profile.birthPlace = `${bio.placeOfBirth}, ${bio.country}`;
|
if (profile.placeOfBirth && bio.country) profile.birthPlace = `${bio.placeOfBirth}, ${bio.country}`;
|
||||||
else if (bio.country) profile.birthPlace = bio.country;
|
else if (bio.country) profile.birthPlace = bio.country;
|
||||||
|
|
|
@ -3,8 +3,11 @@
|
||||||
const Promise = require('bluebird');
|
const Promise = require('bluebird');
|
||||||
const util = require('util');
|
const util = require('util');
|
||||||
const { JSDOM } = require('jsdom');
|
const { JSDOM } = require('jsdom');
|
||||||
const moment = require('moment');
|
const dayjs = require('dayjs');
|
||||||
const format = require('template-format');
|
const format = require('template-format');
|
||||||
|
const utc = require('dayjs/plugin/utc');
|
||||||
|
|
||||||
|
dayjs.extend(utc);
|
||||||
|
|
||||||
const logger = require('../logger')(__filename);
|
const logger = require('../logger')(__filename);
|
||||||
const qu = require('../utils/qu');
|
const qu = require('../utils/qu');
|
||||||
|
@ -241,7 +244,7 @@ async function scrapeApiReleases(json, site) {
|
||||||
if (site.parameters?.scene) release.url = `${site.parameters.scene}${release.path}`;
|
if (site.parameters?.scene) release.url = `${site.parameters.scene}${release.path}`;
|
||||||
else if (site.url && site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`;
|
else if (site.url && site.parameters?.scene !== false) release.url = `${site.url}/en/video${release.path}`;
|
||||||
|
|
||||||
release.date = moment.utc(scene.release_date, 'YYYY-MM-DD').toDate();
|
release.date = dayjs.utc(scene.release_date, 'YYYY-MM-DD').toDate();
|
||||||
release.director = scene.directors[0]?.name || null;
|
release.director = scene.directors[0]?.name || null;
|
||||||
|
|
||||||
release.actors = scene.actors.map((actor) => ({
|
release.actors = scene.actors.map((actor) => ({
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const moment = require('moment');
|
const dayjs = require('dayjs');
|
||||||
|
|
||||||
const qu = require('../utils/q');
|
const qu = require('../utils/q');
|
||||||
const slugify = require('../utils/slugify');
|
const slugify = require('../utils/slugify');
|
||||||
|
@ -202,7 +202,7 @@ async function scrapeScene({ query, html }, url, channel) {
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchLatest(channel, page = 1) {
|
async function fetchLatest(channel, page = 1) {
|
||||||
const year = moment().subtract(page - 1, ' year').year();
|
const year = dayjs().subtract(page - 1, ' year').year();
|
||||||
|
|
||||||
const url = `${channel.url}/Collections/Date/${year}`;
|
const url = `${channel.url}/Collections/Date/${year}`;
|
||||||
const res = await qu.getAll(url, '.collectionGridLayout li');
|
const res = await qu.getAll(url, '.collectionGridLayout li');
|
||||||
|
|
|
@ -4,7 +4,10 @@ const util = require('util');
|
||||||
const Promise = require('bluebird');
|
const Promise = require('bluebird');
|
||||||
const cheerio = require('cheerio');
|
const cheerio = require('cheerio');
|
||||||
const { JSDOM } = require('jsdom');
|
const { JSDOM } = require('jsdom');
|
||||||
const moment = require('moment');
|
const dayjs = require('dayjs');
|
||||||
|
const utc = require('dayjs/plugin/utc');
|
||||||
|
|
||||||
|
dayjs.extend(utc);
|
||||||
|
|
||||||
const qu = require('../utils/qu');
|
const qu = require('../utils/qu');
|
||||||
const http = require('../utils/http');
|
const http = require('../utils/http');
|
||||||
|
@ -220,7 +223,7 @@ function scrapeUpcoming(html, site) {
|
||||||
.trim()
|
.trim()
|
||||||
.split(', ');
|
.split(', ');
|
||||||
|
|
||||||
release.date = moment
|
release.date = dayjs
|
||||||
.utc($(element).find('.update_date_comingsoon').text().slice(7), 'MM/DD/YYYY')
|
.utc($(element).find('.update_date_comingsoon').text().slice(7), 'MM/DD/YYYY')
|
||||||
.toDate();
|
.toDate();
|
||||||
|
|
||||||
|
|
|
@ -4,7 +4,7 @@
|
||||||
const Promise = require('bluebird');
|
const Promise = require('bluebird');
|
||||||
const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
|
const { CookieJar } = Promise.promisifyAll(require('tough-cookie'));
|
||||||
const cookie = require('cookie');
|
const cookie = require('cookie');
|
||||||
const moment = require('moment');
|
const dayjs = require('dayjs');
|
||||||
|
|
||||||
const qu = require('../utils/qu');
|
const qu = require('../utils/qu');
|
||||||
const slugify = require('../utils/slugify');
|
const slugify = require('../utils/slugify');
|
||||||
|
@ -301,7 +301,7 @@ async function fetchLatest(site, page = 1, options) {
|
||||||
? options.beforeNetwork
|
? options.beforeNetwork
|
||||||
: await getSession(site, options.parameters, url);
|
: await getSession(site, options.parameters, url);
|
||||||
|
|
||||||
const beforeDate = moment().add('1', 'day').format('YYYY-MM-DD');
|
const beforeDate = dayjs().add('1', 'day').format('YYYY-MM-DD');
|
||||||
const limit = 24;
|
const limit = 24;
|
||||||
const apiUrl = site.parameters?.native || site.parameters?.extract
|
const apiUrl = site.parameters?.native || site.parameters?.extract
|
||||||
? `https://site-api.project1service.com/v2/releases?dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`
|
? `https://site-api.project1service.com/v2/releases?dateReleased=<${beforeDate}&limit=${limit}&offset=${limit * (page - 1)}&orderBy=-dateReleased&type=scene`
|
||||||
|
|
|
@ -2,7 +2,10 @@
|
||||||
|
|
||||||
/* eslint-disable newline-per-chained-call */
|
/* eslint-disable newline-per-chained-call */
|
||||||
const cheerio = require('cheerio');
|
const cheerio = require('cheerio');
|
||||||
const moment = require('moment');
|
const dayjs = require('dayjs');
|
||||||
|
const utc = require('dayjs/plugin/utc');
|
||||||
|
|
||||||
|
dayjs.extend(utc);
|
||||||
|
|
||||||
const http = require('../utils/http');
|
const http = require('../utils/http');
|
||||||
const slugify = require('../utils/slugify');
|
const slugify = require('../utils/slugify');
|
||||||
|
@ -29,7 +32,7 @@ function scrapeLatest(html, site) {
|
||||||
const url = `${protocol}//${hostname}${pathname}`;
|
const url = `${protocol}//${hostname}${pathname}`;
|
||||||
const { title, entryId } = titleExtractor(pathname);
|
const { title, entryId } = titleExtractor(pathname);
|
||||||
|
|
||||||
const date = moment.utc(element.find('.entry-date').text(), 'MMM D, YYYY').toDate();
|
const date = dayjs.utc(element.find('.entry-date').text(), 'MMM D, YYYY').toDate();
|
||||||
const actors = element.find('.contain-actors a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
|
const actors = element.find('.contain-actors a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
|
||||||
|
|
||||||
const duration = Number(element.find('.scene-runtime').text().slice(0, -4)) * 60;
|
const duration = Number(element.find('.scene-runtime').text().slice(0, -4)) * 60;
|
||||||
|
@ -62,7 +65,7 @@ function scrapeScene(html, url, site) {
|
||||||
const title = sceneElement.find('h1.scene-title').text();
|
const title = sceneElement.find('h1.scene-title').text();
|
||||||
const description = sceneElement.find('.synopsis').contents().slice(2).text().replace(/[\s\n]+/g, ' ').trim();
|
const description = sceneElement.find('.synopsis').contents().slice(2).text().replace(/[\s\n]+/g, ' ').trim();
|
||||||
|
|
||||||
const date = moment.utc(sceneElement.find('span.entry-date').text()?.match(/\w+ \d{1,2}, \d{4}/), 'MMM D, YYYY').toDate();
|
const date = dayjs.utc(sceneElement.find('span.entry-date').text()?.match(/\w+ \d{1,2}, \d{4}/), 'MMM D, YYYY').toDate();
|
||||||
const actors = $('.performer-list a, h1 a.scene-title').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
|
const actors = $('.performer-list a, h1 a.scene-title').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
|
||||||
|
|
||||||
const duration = Number(sceneElement.find('.duration-ratings .duration').text().slice(10, -4)) * 60;
|
const duration = Number(sceneElement.find('.duration-ratings .duration').text().slice(10, -4)) * 60;
|
||||||
|
|
|
@ -1,7 +1,10 @@
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const { JSDOM } = require('jsdom');
|
const { JSDOM } = require('jsdom');
|
||||||
const moment = require('moment');
|
const dayjs = require('dayjs');
|
||||||
|
const utc = require('dayjs/plugin/utc');
|
||||||
|
|
||||||
|
dayjs.extend(utc);
|
||||||
|
|
||||||
const http = require('../utils/http');
|
const http = require('../utils/http');
|
||||||
|
|
||||||
|
@ -31,8 +34,8 @@ async function scrapeProfile(html, _url, actorName) {
|
||||||
|
|
||||||
if (descriptionString) profile.description = descriptionString.textContent;
|
if (descriptionString) profile.description = descriptionString.textContent;
|
||||||
|
|
||||||
if (bio.Birthday && !/-0001/.test(bio.Birthday)) profile.birthdate = moment.utc(bio.Birthday, 'MMM D, YYYY').toDate(); // birthyear sometimes -0001, see Spencer Bradley as of january 2020
|
if (bio.Birthday && !/-0001/.test(bio.Birthday)) profile.birthdate = dayjs.utc(bio.Birthday, 'MMM D, YYYY').toDate(); // birthyear sometimes -0001, see Spencer Bradley as of january 2020
|
||||||
if (bio.Born) profile.birthdate = moment.utc(bio.Born, 'YYYY-MM-DD').toDate();
|
if (bio.Born) profile.birthdate = dayjs.utc(bio.Born, 'YYYY-MM-DD').toDate();
|
||||||
|
|
||||||
profile.birthPlace = bio['Birth Place'] || bio.Birthplace;
|
profile.birthPlace = bio['Birth Place'] || bio.Birthplace;
|
||||||
profile.residencePlace = bio['City and Country'];
|
profile.residencePlace = bio['City and Country'];
|
||||||
|
|
|
@ -2,7 +2,10 @@
|
||||||
|
|
||||||
/* eslint-disable newline-per-chained-call */
|
/* eslint-disable newline-per-chained-call */
|
||||||
const cheerio = require('cheerio');
|
const cheerio = require('cheerio');
|
||||||
const moment = require('moment');
|
const dayjs = require('dayjs');
|
||||||
|
const utc = require('dayjs/plugin/utc');
|
||||||
|
|
||||||
|
dayjs.extend(utc);
|
||||||
|
|
||||||
const { get, geta } = require('../utils/q');
|
const { get, geta } = require('../utils/q');
|
||||||
const slugify = require('../utils/slugify');
|
const slugify = require('../utils/slugify');
|
||||||
|
@ -35,7 +38,7 @@ function scrapeLatest(html, site) {
|
||||||
const titleText = thumbnailElement.attr('alt');
|
const titleText = thumbnailElement.attr('alt');
|
||||||
const title = titleText.slice(titleText.indexOf(':') + 1).trim();
|
const title = titleText.slice(titleText.indexOf(':') + 1).trim();
|
||||||
|
|
||||||
const date = moment.utc($(element).find('.scene-date'), ['MM/DD/YYYY', 'YYYY-MM-DD']).toDate();
|
const date = dayjs.utc($(element).find('.scene-date'), ['MM/DD/YYYY', 'YYYY-MM-DD']).toDate();
|
||||||
|
|
||||||
const actors = $(element).find('.scene-models a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
|
const actors = $(element).find('.scene-models a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
|
||||||
const likes = Number($(element).find('.scene-votes').text());
|
const likes = Number($(element).find('.scene-votes').text());
|
||||||
|
@ -70,7 +73,7 @@ async function scrapeScene(html, url, site) {
|
||||||
release.title = $('.video-wrapper meta[itemprop="name"]').attr('content');
|
release.title = $('.video-wrapper meta[itemprop="name"]').attr('content');
|
||||||
release.description = $('.video-wrapper meta[itemprop="description"]').attr('content');
|
release.description = $('.video-wrapper meta[itemprop="description"]').attr('content');
|
||||||
|
|
||||||
release.date = moment.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate();
|
release.date = dayjs.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate();
|
||||||
release.actors = $('.content-wrapper .scene-models-list a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
|
release.actors = $('.content-wrapper .scene-models-list a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
|
||||||
|
|
||||||
const timestamp = $('.video-wrapper meta[itemprop="duration"]').attr('content');
|
const timestamp = $('.video-wrapper meta[itemprop="duration"]').attr('content');
|
||||||
|
|
|
@ -1,6 +1,9 @@
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const moment = require('moment');
|
const dayjs = require('dayjs');
|
||||||
|
const utc = require('dayjs/plugin/utc');
|
||||||
|
|
||||||
|
dayjs.extend(utc);
|
||||||
|
|
||||||
const logger = require('../logger')(__filename);
|
const logger = require('../logger')(__filename);
|
||||||
const http = require('../utils/http');
|
const http = require('../utils/http');
|
||||||
|
@ -22,7 +25,7 @@ function scrapeAll(scenes, entity) {
|
||||||
}
|
}
|
||||||
|
|
||||||
release.description = scene.description;
|
release.description = scene.description;
|
||||||
release.date = moment.utc(scene.year, 'YYYY').toDate();
|
release.date = dayjs.utc(scene.year, 'YYYY').toDate();
|
||||||
release.datePrecision = 'year';
|
release.datePrecision = 'year';
|
||||||
|
|
||||||
release.actors = scene.actors.map((actor) => ({
|
release.actors = scene.actors.map((actor) => ({
|
||||||
|
|
|
@ -5,7 +5,7 @@
|
||||||
const config = require('config');
|
const config = require('config');
|
||||||
const faker = require('faker');
|
const faker = require('faker');
|
||||||
const { nanoid } = require('nanoid');
|
const { nanoid } = require('nanoid');
|
||||||
const moment = require('moment');
|
const dayjs = require('dayjs');
|
||||||
|
|
||||||
const knex = require('../knex');
|
const knex = require('../knex');
|
||||||
const capitalize = require('../utils/capitalize');
|
const capitalize = require('../utils/capitalize');
|
||||||
|
@ -236,7 +236,7 @@ async function fetchLatest(entity, page, options) {
|
||||||
const release = {};
|
const release = {};
|
||||||
|
|
||||||
release.entryId = nanoid();
|
release.entryId = nanoid();
|
||||||
release.date = moment().subtract(Math.floor(Math.random() * index), 'days').toDate();
|
release.date = dayjs().subtract(Math.floor(Math.random() * index), 'days').toDate();
|
||||||
|
|
||||||
if (options.source) {
|
if (options.source) {
|
||||||
// select from configured random image source
|
// select from configured random image source
|
||||||
|
|
|
@ -2,7 +2,10 @@
|
||||||
|
|
||||||
/* eslint-disable newline-per-chained-call */
|
/* eslint-disable newline-per-chained-call */
|
||||||
const Promise = require('bluebird');
|
const Promise = require('bluebird');
|
||||||
const moment = require('moment');
|
const dayjs = require('dayjs');
|
||||||
|
const utc = require('dayjs/plugin/utc');
|
||||||
|
|
||||||
|
dayjs.extend(utc);
|
||||||
|
|
||||||
const qu = require('../utils/qu');
|
const qu = require('../utils/qu');
|
||||||
const http = require('../utils/http');
|
const http = require('../utils/http');
|
||||||
|
@ -172,7 +175,7 @@ function scrapeUpcoming(scene, site) {
|
||||||
.map((component) => `${component.charAt(0).toUpperCase()}${component.slice(1)}`)
|
.map((component) => `${component.charAt(0).toUpperCase()}${component.slice(1)}`)
|
||||||
.join(' ');
|
.join(' ');
|
||||||
|
|
||||||
release.date = moment.utc(scene.releaseDate).toDate();
|
release.date = dayjs.utc(scene.releaseDate).toDate();
|
||||||
release.datePrecision = 'minute';
|
release.datePrecision = 'minute';
|
||||||
|
|
||||||
release.actors = scene.models.map((model) => model.name);
|
release.actors = scene.models.map((model) => model.name);
|
||||||
|
|
|
@ -1,7 +1,10 @@
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const { JSDOM } = require('jsdom');
|
const { JSDOM } = require('jsdom');
|
||||||
const moment = require('moment');
|
const dayjs = require('dayjs');
|
||||||
|
const utc = require('dayjs/plugin/utc');
|
||||||
|
|
||||||
|
dayjs.extend(utc);
|
||||||
|
|
||||||
const http = require('../utils/http');
|
const http = require('../utils/http');
|
||||||
|
|
||||||
|
@ -17,7 +20,7 @@ function scrapeLatest(html, site) {
|
||||||
release.url = `${origin}${scene.querySelector(':scope > a').href}`;
|
release.url = `${origin}${scene.querySelector(':scope > a').href}`;
|
||||||
release.entryId = scene.dataset.videoId;
|
release.entryId = scene.dataset.videoId;
|
||||||
release.title = scene.querySelector('.card-title').textContent;
|
release.title = scene.querySelector('.card-title').textContent;
|
||||||
release.date = moment.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate();
|
release.date = dayjs.utc(scene.dataset.date, 'MMMM DD, YYYY').toDate();
|
||||||
release.actors = Array.from(scene.querySelectorAll('.actors a'), (el) => el.textContent);
|
release.actors = Array.from(scene.querySelectorAll('.actors a'), (el) => el.textContent);
|
||||||
|
|
||||||
// slow CDN?
|
// slow CDN?
|
||||||
|
@ -68,7 +71,7 @@ function scrapeScene(html, site, url) {
|
||||||
const durationEls = Array.from(scene.querySelectorAll('#t2019-stime span'));
|
const durationEls = Array.from(scene.querySelectorAll('#t2019-stime span'));
|
||||||
|
|
||||||
if (durationEls.length > 1) {
|
if (durationEls.length > 1) {
|
||||||
release.date = moment.utc(durationEls[0].textContent, 'MMMM DD, YYYY').toDate();
|
release.date = dayjs.utc(durationEls[0].textContent, 'MMMM DD, YYYY').toDate();
|
||||||
release.duration = Number(durationEls[1].textContent.match(/\d+/)[0]) * 60;
|
release.duration = Number(durationEls[1].textContent.match(/\d+/)[0]) * 60;
|
||||||
} else {
|
} else {
|
||||||
release.duration = Number(durationEls[0].textContent.match(/\d+/)[0]) * 60;
|
release.duration = Number(durationEls[0].textContent.match(/\d+/)[0]) * 60;
|
||||||
|
|
|
@ -2,7 +2,7 @@
|
||||||
|
|
||||||
const config = require('config');
|
const config = require('config');
|
||||||
const Promise = require('bluebird');
|
const Promise = require('bluebird');
|
||||||
const moment = require('moment');
|
const dayjs = require('dayjs');
|
||||||
|
|
||||||
const argv = require('./argv');
|
const argv = require('./argv');
|
||||||
const logger = require('./logger')(__filename);
|
const logger = require('./logger')(__filename);
|
||||||
|
@ -99,7 +99,7 @@ function needNextPage(pageReleases, accReleases, isUpcoming, unextracted = []) {
|
||||||
.sort((releaseA, releaseB) => releaseB.date - releaseA.date)
|
.sort((releaseA, releaseB) => releaseB.date - releaseA.date)
|
||||||
.slice(-1)[0];
|
.slice(-1)[0];
|
||||||
|
|
||||||
if (moment(oldestReleaseOnPage.date).isAfter(argv.after)) {
|
if (dayjs(oldestReleaseOnPage.date).isAfter(argv.after)) {
|
||||||
// oldest release on page is newer than the specified date cut-off
|
// oldest release on page is newer than the specified date cut-off
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
@ -149,7 +149,7 @@ async function scrapeReleases(scraper, entity, preData, isUpcoming) {
|
||||||
const hasDates = releases.every((release) => !!release.date);
|
const hasDates = releases.every((release) => !!release.date);
|
||||||
|
|
||||||
const limitedReleases = (argv.last && releases.slice(0, Math.max(argv.last, 0)))
|
const limitedReleases = (argv.last && releases.slice(0, Math.max(argv.last, 0)))
|
||||||
|| (hasDates && releases.filter((release) => moment(release.date).isAfter(argv.after)))
|
|| (hasDates && releases.filter((release) => dayjs(release.date).isAfter(argv.after)))
|
||||||
|| releases.slice(0, Math.max(argv.missingDateLimit, 0));
|
|| releases.slice(0, Math.max(argv.missingDateLimit, 0));
|
||||||
|
|
||||||
const { uniqueReleases, duplicateReleases } = argv.force
|
const { uniqueReleases, duplicateReleases } = argv.force
|
||||||
|
|
|
@ -3,7 +3,10 @@
|
||||||
const config = require('config');
|
const config = require('config');
|
||||||
const path = require('path');
|
const path = require('path');
|
||||||
const fs = require('fs-extra');
|
const fs = require('fs-extra');
|
||||||
const moment = require('moment');
|
const dayjs = require('dayjs');
|
||||||
|
const utc = require('dayjs/plugin/utc');
|
||||||
|
|
||||||
|
dayjs.extend(utc);
|
||||||
|
|
||||||
const argv = require('../argv');
|
const argv = require('../argv');
|
||||||
const knex = require('../knex');
|
const knex = require('../knex');
|
||||||
|
@ -25,7 +28,8 @@ async function actorPosters(actorNames) {
|
||||||
const source = path.join(config.media.path, poster.path);
|
const source = path.join(config.media.path, poster.path);
|
||||||
|
|
||||||
const directory = path.join(config.media.path, 'extracted', poster.actor_name);
|
const directory = path.join(config.media.path, 'extracted', poster.actor_name);
|
||||||
const target = path.join(directory, `${poster.actor_name} - ${poster.network_name}: ${poster.site_name} - ${poster.title.replace(/[/.]/g, '_') || poster.actor_name} (${moment.utc(poster.date).format('YYYY-MM-DD')})-${poster.index}.jpeg`);
|
const formattedDate = dayjs.utc(poster.date).format('YYYY-MM-DD');
|
||||||
|
const target = path.join(directory, `${poster.actor_name} - ${poster.network_name}: ${poster.site_name} - ${poster.title.replace(/[/.]/g, '_') || poster.actor_name} (${formattedDate})-${poster.index}.jpeg`);
|
||||||
await fs.mkdir(path.join(directory), { recursive: true });
|
await fs.mkdir(path.join(directory), { recursive: true });
|
||||||
|
|
||||||
const file = await fs.readFile(source);
|
const file = await fs.readFile(source);
|
||||||
|
@ -50,7 +54,7 @@ async function sitePosters(siteSlugs) {
|
||||||
const directory = path.join(config.media.path, 'extracted', poster.site_name);
|
const directory = path.join(config.media.path, 'extracted', poster.site_name);
|
||||||
|
|
||||||
const source = path.join(config.media.path, poster.path);
|
const source = path.join(config.media.path, poster.path);
|
||||||
const target = path.join(directory, `${poster.site_name} - ${moment.utc(poster.date).format('YYYY-MM-DD')} - ${poster.title.replace(/[/.]/g, '_')}.jpeg`);
|
const target = path.join(directory, `${poster.site_name} - ${dayjs.utc(poster.date).format('YYYY-MM-DD')} - ${poster.title.replace(/[/.]/g, '_')}.jpeg`);
|
||||||
|
|
||||||
await fs.mkdir(directory, { recursive: true });
|
await fs.mkdir(directory, { recursive: true });
|
||||||
await fs.copyFile(source, target);
|
await fs.copyFile(source, target);
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const { JSDOM } = require('jsdom');
|
const { JSDOM } = require('jsdom');
|
||||||
const moment = require('moment');
|
const dayjs = require('dayjs');
|
||||||
const http = require('./http');
|
const http = require('./http');
|
||||||
const virtualConsole = require('./virtual-console')(__filename);
|
const virtualConsole = require('./virtual-console')(__filename);
|
||||||
|
|
||||||
|
@ -38,10 +38,10 @@ function extractDate(dateString, format, match) {
|
||||||
|
|
||||||
function formatDate(dateValue, format, inputFormat) {
|
function formatDate(dateValue, format, inputFormat) {
|
||||||
if (inputFormat) {
|
if (inputFormat) {
|
||||||
return moment(dateValue, inputFormat).format(format);
|
return dayjs(dateValue, inputFormat).format(format);
|
||||||
}
|
}
|
||||||
|
|
||||||
return moment(dateValue).format(format);
|
return dayjs(dateValue).format(format);
|
||||||
}
|
}
|
||||||
|
|
||||||
function durationToSeconds(durationString, match) {
|
function durationToSeconds(durationString, match) {
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
const { makeExtendSchemaPlugin, gql } = require('graphile-utils');
|
const { makeExtendSchemaPlugin, gql } = require('graphile-utils');
|
||||||
const moment = require('moment');
|
const dayjs = require('dayjs');
|
||||||
const { cmToFeetInches, cmToInches, kgToLbs } = require('../../utils/convert');
|
const { cmToFeetInches, cmToInches, kgToLbs } = require('../../utils/convert');
|
||||||
|
|
||||||
const schemaExtender = makeExtendSchemaPlugin((_build) => ({
|
const schemaExtender = makeExtendSchemaPlugin((_build) => ({
|
||||||
|
@ -45,12 +45,12 @@ const schemaExtender = makeExtendSchemaPlugin((_build) => ({
|
||||||
ageFromBirth(parent, _args, _context, _info) {
|
ageFromBirth(parent, _args, _context, _info) {
|
||||||
if (!parent.dateOfBirth) return null;
|
if (!parent.dateOfBirth) return null;
|
||||||
|
|
||||||
return moment().diff(parent.dateOfBirth, 'years');
|
return dayjs().diff(parent.dateOfBirth, 'years');
|
||||||
},
|
},
|
||||||
ageAtDeath(parent, _args, _context, _info) {
|
ageAtDeath(parent, _args, _context, _info) {
|
||||||
if (!parent.dateOfDeath) return null;
|
if (!parent.dateOfDeath) return null;
|
||||||
|
|
||||||
return moment(parent.dateOfDeath).diff(parent.dateOfBirth, 'years');
|
return dayjs(parent.dateOfDeath).diff(parent.dateOfBirth, 'years');
|
||||||
},
|
},
|
||||||
height(parent, args, _context, _info) {
|
height(parent, args, _context, _info) {
|
||||||
if (!parent.height) return null;
|
if (!parent.height) return null;
|
||||||
|
|
Loading…
Reference in New Issue