DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 39 Pull Requests 1 Releases Wiki Activity

Added description property to recent scrapers (apparently forgotten). Reconfigured eslint for scrapers.

This commit is contained in:
ThePendulum 2019-04-07 05:19:44 +02:00
parent 4b19b49e6c
commit 6b9d99f495
9 changed files with 21 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.eslintignore Normal file
View File

@ -0,0 +1 @@
src/scrapers/template.js

2
.eslintrc
View File

@ -3,7 +3,7 @@
"parser": "babel-eslint",
"extends": "airbnb-base",
"parserOptions": {
"sourceType": "script"
"sourceType": "script",
},
"rules": {
"strict": 0,

5
src/scrapers/blowpass.js
View File

@ -1,6 +1,6 @@
'use strict';
/* eslint-disable */
/* eslint-disable newline-per-chained-call */
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
@ -47,7 +47,7 @@ async function scrapeScene(html, url, site) {
const shootId = workName.length > 1 ? workName[0] : null;
const entryId = url.split('/').slice(-1)[0];
const title = data.name;
const description = data.description;
const { description } = data;
const date = moment.utc(data.isPartOf.datePublished, 'YYYY-MM-DD').toDate();
// const actors = sceneElement.find('.sceneActors a').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray();
@ -73,6 +73,7 @@ async function scrapeScene(html, url, site) {
shootId,
entryId,
title,
description,
actors,
date,
duration,

5
src/scrapers/brazzers.js
View File

@ -1,6 +1,6 @@
'use strict';
/* eslint-disable */
/* eslint-disable newline-per-chained-call */
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
@ -83,12 +83,15 @@ async function scrapeScene(html, url, site) {
url,
shootId,
title,
description,
actors,
date,
duration,
rating: {
likes,
dislikes,
},
tags,
site: channelSite || site,
};
}

3
src/scrapers/ddfnetwork.js
View File

@ -1,6 +1,5 @@
'use strict';
/* eslint-disable */
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
@ -8,6 +7,7 @@ const moment = require('moment');
const knex = require('../knex');
const { matchTags } = require('../tags');
/* eslint-disable newline-per-chained-call */
function scrapeLatest(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.card.m-1').toArray();
@ -62,6 +62,7 @@ async function scrapeScene(html, url, site) {
url: channelSite ? `${channelSite.url}${new URL(url).pathname}` : url,
entryId,
title,
description,
actors,
date,
duration,

5
src/scrapers/mofos.js
View File

@ -1,6 +1,5 @@
'use strict';
/* eslint-disable */
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
@ -8,6 +7,7 @@ const moment = require('moment');
const knex = require('../knex');
const { matchTags } = require('../tags');
/* eslint-disable newline-per-chained-call */
function scrape(html, site) {
const $ = cheerio.load(html, { normalizeWhitespace: true });
const sceneElements = $('.widget-release-card').toArray();
@ -69,12 +69,13 @@ async function scrapeScene(html, url, site) {
url,
entryId,
title,
description,
actors,
tags,
rating: {
stars,
},
site,
site: channelSite || site,
};
}

4
src/scrapers/private.js
View File

@ -1,6 +1,6 @@
'use strict';
/* eslint-disable */
/* eslint-disable newline-per-chained-call */
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
@ -44,7 +44,7 @@ async function scrapeScene(html, url, site) {
const shootId = url.split('/').slice(-1)[0];
const title = $('.video-wrapper meta[itemprop="name"]').attr('content');
const date = moment.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate()
const date = moment.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate();
const actors = $('.content-wrapper .scene-models-list a').map((actorIndex, actorElement) => $(actorElement).text()).toArray();
const description = $('.video-wrapper meta[itemprop="description"]').attr('content');

3
src/scrapers/template.js
View File

@ -1,6 +1,6 @@
'use strict';
/* eslint-disable */
/* eslint-disable newline-per-chained-call */
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
@ -68,6 +68,7 @@ async function scrapeScene(html, url, site) {
url,
shootId,
title,
description,
actors,
director,
date,

6
src/scrapers/vixen.js
View File

@ -1,6 +1,6 @@
'use strict';
/* eslint-disable */
/* eslint-disable newline-per-chained-call */
const bhttp = require('bhttp');
const cheerio = require('cheerio');
const moment = require('moment');
@ -15,7 +15,7 @@ function scrapeLatest(html, site) {
return scenes.map((scene) => {
const shootId = String(scene.newId);
const title = scene.title;
const { title } = scene;
const url = `${site.url}${scene.targetUrl}`;
const date = moment.utc(scene.releaseDateFormatted, 'MMMM DD, YYYY').toDate();
const actors = scene.models;
@ -45,7 +45,7 @@ async function scrapeScene(html, url, site) {
const scene = data.page.data[`${pathname}${search}`].data.video;
const shootId = String(scene.newId);
const title = scene.title;
const { title } = scene;
const date = new Date(scene.releaseDate);
const actors = scene.models;
const stars = scene.totalRateVal;