DebaucheryLibrarian/traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added parameters column to sites database, fixes Perv City scraper. Getting shoot ID from all existing scrapers.

Browse Source
This commit is contained in:
ThePendulum
2019-03-26 01:26:47 +01:00
parent 6a20cbc721
commit 8421cd8648
11 changed files with 45 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/scrapers/kink.js
View File

@@ -15,7 +15,7 @@ function scrapeLatest(html, site) {
const sceneLinkElement = $(element).find('.shoot-thumb-title a');
const href = sceneLinkElement.attr('href');
const url = `https://kink.com${href}`;
const id = href.split('/')[2];
const shootId = href.split('/')[2];
const title = sceneLinkElement.text();
const date = moment.utc($(element).find('.date').text(), 'MMM DD, YYYY').toDate();
@@ -28,7 +28,7 @@ function scrapeLatest(html, site) {
return {
url,
id,
shootId,
title,
actors,
date,
@@ -41,7 +41,7 @@ function scrapeLatest(html, site) {
});
}
async function scrapeScene(html, url, id, ratingRes, site) {
async function scrapeScene(html, url, shootId, ratingRes, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
// const title = $('h1.shoot-title').text().replace(/\ue800/, ''); // fallback, special character is 'like'-heart
@@ -70,7 +70,7 @@ async function scrapeScene(html, url, id, ratingRes, site) {
return {
url,
id,
shootId,
title,
date,
actors,
@@ -90,14 +90,14 @@ async function fetchLatest(site) {
}
async function fetchScene(url, site) {
const id = new URL(url).pathname.split('/')[2];
const shootId = new URL(url).pathname.split('/')[2];
const [res, ratingRes] = await Promise.all([
bhttp.get(url),
bhttp.get(`https://kink.com/api/ratings/${id}`),
bhttp.get(`https://kink.com/api/ratings/${shootId}`),
]);
return scrapeScene(res.body.toString(), url, id, ratingRes, site);
return scrapeScene(res.body.toString(), url, shootId, ratingRes, site);
}
module.exports = {