boiii
/
traxxx
forked from DebaucheryLibrarian/traxxx
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fetching old scene links from Vivid search API.

This commit is contained in:
ThePendulum 2020-02-11 19:05:12 +01:00
parent 4caf0dbe95
commit df4a2b0bb3
2 changed files with 47 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
seeds/01_sites.js
View File

@ -4454,36 +4454,48 @@ const sites = [
name: 'Vivid Celeb',
url: 'https://www.vividceleb.com',
network: 'vivid',
parameters: {
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2018-03-25'),
},
},
{
slug: 'thebrats',
name: 'The Brats',
url: 'https://www.thebrats.com',
network: 'vivid',
parameters: {
useGamma: true,
},
},
{
slug: 'wheretheboysarent',
name: 'Where The Boys Aren\'t',
url: 'https://www.wheretheboysarent.com',
network: 'vivid',
parameters: {
useGamma: true,
},
},
{
slug: 'nineteen',
name: 'Nineteen',
url: 'https://www.nineteen.com',
network: 'vivid',
parameters: {
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2019-01-23'),
},
},
{
slug: 'nastystepfamily',
name: 'Nasty Step Family',
url: 'https://www.nastystepfamily.com',
network: 'vivid',
parameters: {
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2019-01-29'),
},
},
{
slug: 'girlswhofuckgirls',
@ -4491,10 +4503,10 @@ const sites = [
url: 'https://www.girlswhofuckgirls.com',
network: 'vivid',
parameters: {
useGamma: true,
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2019-05-21'),
},
},
{
@ -4503,10 +4515,10 @@ const sites = [
url: 'https://www.petited.com',
network: 'vivid',
parameters: {
useGamma: true,
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2019-01-28'),
},
},
{
@ -4515,10 +4527,10 @@ const sites = [
url: 'https://www.orgytrain.com',
network: 'vivid',
parameters: {
useGamma: true,
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2019-01-09'),
},
},
{
@ -4527,10 +4539,10 @@ const sites = [
url: 'https://www.momisamilf.com',
network: 'vivid',
parameters: {
useGamma: true,
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2019-01-25'),
},
},
{
@ -4539,10 +4551,10 @@ const sites = [
url: 'https://www.blackwhitefuckfest.com',
network: 'vivid',
parameters: {
useGamma: true,
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2019-01-30'),
},
},
{
@ -4551,10 +4563,10 @@ const sites = [
url: 'https://www.65inchhugeasses.com',
network: 'vivid',
parameters: {
useGamma: true,
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2019-05-18'),
},
},
{
@ -4563,10 +4575,10 @@ const sites = [
url: 'https://www.brandnewfaces.com',
network: 'vivid',
parameters: {
useGamma: true,
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2018-02-28'),
},
},
{
@ -4575,10 +4587,10 @@ const sites = [
url: 'https://www.vividclassic.com',
network: 'vivid',
parameters: {
useGamma: true,
referer: 'https://www.thebrats.com',
deep: 'https://www.thebrats.com/en/video',
scene: false,
lastNative: new Date('2016-06-29'),
},
},
// VIXEN

21
src/scrapers/vivid.js
View File

@ -5,6 +5,7 @@ const bhttp = require('bhttp');
const { get, date } = require('../utils/q');
const { fetchApiLatest, fetchApiUpcoming, fetchScene, fetchApiProfile } = require('./gamma');
const slugify = require('../utils/slugify');
function scrapeLatestNative(scenes, site) {
return scenes.map((scene) => {
@ -103,6 +104,26 @@ async function fetchSceneNative(url, site, release) {
async function fetchSceneWrapper(url, site, release) {
const scene = await fetchScene(url, site, release);
if (scene.date - new Date(site.parameters.lastNative) <= 0) {
// scene is probably still available on Vivid site, use search API to get URL and original date
const searchUrl = `${site.url}/videos/api/?limit=10&sort=datedesc&search=${encodeURI(scene.title)}`;
const searchRes = await bhttp.get(searchUrl, {
decodeJSON: true,
});
if (searchRes.statusCode === 200 && searchRes.body.code === 200) {
const sceneMatch = searchRes.body.responseData.find(item => slugify(item.name) === slugify(scene.title));
if (sceneMatch) {
return {
...scene,
url: `${site.url}${sceneMatch.url}`,
date: date(sceneMatch.release_date, 'YYYY-MM-DD'),
};
}
}
}
return scene;
}