DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Scrapers can now iterate through pages. Filtering unique releases before saving to database. Improved scrapers and rendering.

Browse Source
This commit is contained in:
ThePendulum
2019-04-05 03:45:40 +02:00
parent cbb4fdc919
commit 2b818e379a
14 changed files with 99 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/scrapers/brazzers.js
View File

@@ -93,8 +93,8 @@ async function scrapeScene(html, url, site) {
};
}
async function fetchLatest(site) {
const res = await bhttp.get(`${site.url}/`);
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`${site.url}/page/${page}/`);
return scrape(res.body.toString(), site, false);
}

4
src/scrapers/julesjordan.js
View File

@@ -107,8 +107,8 @@ async function scrapeScene(html, url, site) {
};
}
async function fetchLatest(site, _date) {
const res = await bhttp.get(`${site.url}/trial/categories/movies_1_d.html`);
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`${site.url}/trial/categories/movies_${page}_d.html`);
return scrapeLatest(res.body.toString(), site);
}

6
src/scrapers/kink.js
View File

@@ -16,7 +16,7 @@ function scrapeLatest(html, site) {
const href = sceneLinkElement.attr('href');
const url = `https://kink.com${href}`;
const shootId = href.split('/')[2];
const title = sceneLinkElement.text();
const title = sceneLinkElement.text().trim();
const date = moment.utc($(element).find('.date').text(), 'MMM DD, YYYY').toDate();
const actors = $(element).find('.shoot-thumb-models a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
@@ -85,8 +85,8 @@ async function scrapeScene(html, url, shootId, ratingRes, site) {
};
}
async function fetchLatest(site) {
const res = await bhttp.get(`${site.url}/latest`);
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`${site.url}/latest/page/${page}`);
return scrapeLatest(res.body.toString(), site);
}

9
src/scrapers/legalporno.js
View File

@@ -23,14 +23,15 @@ function scrapeLatest(html, site) {
const sceneLinkElement = $(element).find('.thumbnail-title a');
const url = sceneLinkElement.attr('href');
const originalTitle = sceneLinkElement.attr('title');
const originalTitle = sceneLinkElement.text().trim(); // title attribute breaks when they use \\ escaping
const { shootId, title } = extractTitle(originalTitle);
const internalId = new URL(url).pathname.split('/')[2];
const date = moment.utc($(element).attr('release'), 'YYYY/MM/DD').toDate();
return {
url,
shootId,
shootId: shootId || internalId,
title,
date,
site,
@@ -68,8 +69,8 @@ async function scrapeScene(html, url, site) {
};
}
async function fetchLatest(site) {
const res = await bhttp.get(`${site.url}/new-videos`);
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`${site.url}/new-videos/${page}`);
return scrapeLatest(res.body.toString(), site);
}

8
src/scrapers/pervcity.js
View File

@@ -33,11 +33,13 @@ function scrape(html, site) {
};
}
async function fetchLatest(site) {
const res = await bhttp.get(`${site.url}/final_latestupdateview.php?limitstart=0&limitend=9&websiteid=0&deviceview=browser&tourId=${site.parameters.tourId}`);
async function fetchLatest(site, page = 1) {
const res = page === 1
? await bhttp.get(`${site.url}/final_latestupdateview.php?limitstart=${(page - 1) * 9}&limitend=9&websiteid=0&deviceview=browser&tourId=${site.parameters.tourId}`)
: await bhttp.get(`${site.url}/final_load_latestupdate_grid_view.php?limitstart=0&limitend=${(page - 1) * 8 + 1}&websiteid=0&deviceview=browser&tourId=${site.parameters.tourId}`);
const elements = JSON.parse(res.body.toString());
const latest = elements.total_arr.map(html => scrape(html, site));
const latest = Object.values(elements.total_arr).map(html => scrape(html, site)); // total_arr is a key-value object for final_load_latestupdate_grid_view.php
return latest;
}

4
src/scrapers/private.js
View File

@@ -85,8 +85,8 @@ async function scrapeScene(html, url, site) {
};
}
async function fetchLatest(site) {
const res = await bhttp.get(`${site.url}/`);
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`${site.url}/${page}/`);
return scrapeLatest(res.body.toString(), site);
}

8
src/scrapers/vixen.js
View File

@@ -14,7 +14,7 @@ function scrapeLatest(html, site) {
const { videos: scenes } = JSON.parse(stateObject.html().trim().slice(27, -1));
return scenes.map((scene) => {
const shootId = scene.newId;
const shootId = String(scene.newId);
const title = scene.title;
const url = `${site.url}${scene.targetUrl}`;
const date = moment.utc(scene.releaseDateFormatted, 'MMMM DD, YYYY').toDate();
@@ -44,7 +44,7 @@ async function scrapeScene(html, url, site) {
const scene = data.page.data[`${pathname}${search}`].data.video;
const shootId = scene.newId;
const shootId = String(scene.newId);
const title = scene.title;
const date = new Date(scene.releaseDate);
const actors = scene.models;
@@ -72,8 +72,8 @@ async function scrapeScene(html, url, site) {
};
}
async function fetchLatest(site) {
const res = await bhttp.get(`${site.url}/videos`);
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`${site.url}/videos?page=${page}&size=7`);
return scrapeLatest(res.body.toString(), site);
}

11
src/scrapers/xempire.js
View File

@@ -121,15 +121,10 @@ async function scrapeScene(html, url, site) {
};
}
async function fetchLatest(site, storedReleases) {
const res = await bhttp.get(`${site.url}/en/videos`);
const releases = scrape(res.body.toString(), site);
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`${site.url}/en/videos/AllCategories/0/${page}`);
const storedShootIds = new Set(storedReleases.map(release => release.shoot_id));
const newReleases = releases.filter(release => !storedShootIds.has(release.shootId));
console.log(newReleases);
return scrape(res.body.toString(), site);
}
async function fetchUpcoming(site) {