DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Scrapers can now iterate through pages. Filtering unique releases before saving to database. Improved scrapers and rendering.

Browse Source
This commit is contained in:
ThePendulum
2019-04-05 03:45:40 +02:00
parent cbb4fdc919
commit 2b818e379a
14 changed files with 99 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src/scrapers/kink.js
View File

@@ -16,7 +16,7 @@ function scrapeLatest(html, site) {
const href = sceneLinkElement.attr('href');
const url = `https://kink.com${href}`;
const shootId = href.split('/')[2];
const title = sceneLinkElement.text();
const title = sceneLinkElement.text().trim();
const date = moment.utc($(element).find('.date').text(), 'MMM DD, YYYY').toDate();
const actors = $(element).find('.shoot-thumb-models a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
@@ -85,8 +85,8 @@ async function scrapeScene(html, url, shootId, ratingRes, site) {
};
}
async function fetchLatest(site) {
const res = await bhttp.get(`${site.url}/latest`);
async function fetchLatest(site, page = 1) {
const res = await bhttp.get(`${site.url}/latest/page/${page}`);
return scrapeLatest(res.body.toString(), site);
}