Added Private scraper. Added Vixen scraper to repository.

This commit is contained in:
2019-04-04 04:00:28 +02:00
parent 439d3225ec
commit b3beeef3e4
14 changed files with 545 additions and 90 deletions

View File

@@ -34,6 +34,7 @@ function scrape(html, site) {
shootId,
title,
actors,
director: 'Mason',
date,
rating: {
likes,
@@ -62,6 +63,7 @@ async function scrapeSceneFallback($, url, site) {
title,
date,
actors,
director: 'Mason',
description,
tags,
rating: {
@@ -108,6 +110,7 @@ async function scrapeScene(html, url, site) {
title,
date,
actors,
director: 'Mason',
description,
duration,
tags,
@@ -118,10 +121,15 @@ async function scrapeScene(html, url, site) {
};
}
async function fetchLatest(site) {
async function fetchLatest(site, storedReleases) {
const res = await bhttp.get(`${site.url}/en/videos`);
const releases = scrape(res.body.toString(), site);
return scrape(res.body.toString(), site);
const storedShootIds = new Set(storedReleases.map(release => release.shoot_id));
const newReleases = releases.filter(release => !storedShootIds.has(release.shootId));
console.log(newReleases);
}
async function fetchUpcoming(site) {