From 6b9d99f495b6a93289c92c555cdad66216a1b056 Mon Sep 17 00:00:00 2001 From: Niels Simenon Date: Sun, 7 Apr 2019 05:19:44 +0200 Subject: [PATCH] Added description property to recent scrapers (apparently forgotten). Reconfigured eslint for scrapers. --- .eslintignore | 1 + .eslintrc | 2 +- src/scrapers/blowpass.js | 5 +++-- src/scrapers/brazzers.js | 5 ++++- src/scrapers/ddfnetwork.js | 3 ++- src/scrapers/mofos.js | 5 +++-- src/scrapers/private.js | 4 ++-- src/scrapers/template.js | 3 ++- src/scrapers/vixen.js | 6 +++--- 9 files changed, 21 insertions(+), 13 deletions(-) create mode 100644 .eslintignore diff --git a/.eslintignore b/.eslintignore new file mode 100644 index 00000000..f4416e34 --- /dev/null +++ b/.eslintignore @@ -0,0 +1 @@ +src/scrapers/template.js diff --git a/.eslintrc b/.eslintrc index 254d8fd7..9f9fb622 100644 --- a/.eslintrc +++ b/.eslintrc @@ -3,7 +3,7 @@ "parser": "babel-eslint", "extends": "airbnb-base", "parserOptions": { - "sourceType": "script" + "sourceType": "script", }, "rules": { "strict": 0, diff --git a/src/scrapers/blowpass.js b/src/scrapers/blowpass.js index 230b8bf9..6442b587 100644 --- a/src/scrapers/blowpass.js +++ b/src/scrapers/blowpass.js @@ -1,6 +1,6 @@ 'use strict'; -/* eslint-disable */ +/* eslint-disable newline-per-chained-call */ const bhttp = require('bhttp'); const cheerio = require('cheerio'); const moment = require('moment'); @@ -47,7 +47,7 @@ async function scrapeScene(html, url, site) { const shootId = workName.length > 1 ? workName[0] : null; const entryId = url.split('/').slice(-1)[0]; const title = data.name; - const description = data.description; + const { description } = data; const date = moment.utc(data.isPartOf.datePublished, 'YYYY-MM-DD').toDate(); // const actors = sceneElement.find('.sceneActors a').map((actorIndex, actorElement) => $(actorElement).text().trim()).toArray(); @@ -73,6 +73,7 @@ async function scrapeScene(html, url, site) { shootId, entryId, title, + description, actors, date, duration, diff --git a/src/scrapers/brazzers.js b/src/scrapers/brazzers.js index aabc9750..7d6aae1c 100644 --- a/src/scrapers/brazzers.js +++ b/src/scrapers/brazzers.js @@ -1,6 +1,6 @@ 'use strict'; -/* eslint-disable */ +/* eslint-disable newline-per-chained-call */ const bhttp = require('bhttp'); const cheerio = require('cheerio'); const moment = require('moment'); @@ -83,12 +83,15 @@ async function scrapeScene(html, url, site) { url, shootId, title, + description, actors, date, + duration, rating: { likes, dislikes, }, + tags, site: channelSite || site, }; } diff --git a/src/scrapers/ddfnetwork.js b/src/scrapers/ddfnetwork.js index 488554bd..5157a3d4 100644 --- a/src/scrapers/ddfnetwork.js +++ b/src/scrapers/ddfnetwork.js @@ -1,6 +1,5 @@ 'use strict'; -/* eslint-disable */ const bhttp = require('bhttp'); const cheerio = require('cheerio'); const moment = require('moment'); @@ -8,6 +7,7 @@ const moment = require('moment'); const knex = require('../knex'); const { matchTags } = require('../tags'); +/* eslint-disable newline-per-chained-call */ function scrapeLatest(html, site) { const $ = cheerio.load(html, { normalizeWhitespace: true }); const sceneElements = $('.card.m-1').toArray(); @@ -62,6 +62,7 @@ async function scrapeScene(html, url, site) { url: channelSite ? `${channelSite.url}${new URL(url).pathname}` : url, entryId, title, + description, actors, date, duration, diff --git a/src/scrapers/mofos.js b/src/scrapers/mofos.js index 116a2e67..7325a942 100644 --- a/src/scrapers/mofos.js +++ b/src/scrapers/mofos.js @@ -1,6 +1,5 @@ 'use strict'; -/* eslint-disable */ const bhttp = require('bhttp'); const cheerio = require('cheerio'); const moment = require('moment'); @@ -8,6 +7,7 @@ const moment = require('moment'); const knex = require('../knex'); const { matchTags } = require('../tags'); +/* eslint-disable newline-per-chained-call */ function scrape(html, site) { const $ = cheerio.load(html, { normalizeWhitespace: true }); const sceneElements = $('.widget-release-card').toArray(); @@ -69,12 +69,13 @@ async function scrapeScene(html, url, site) { url, entryId, title, + description, actors, tags, rating: { stars, }, - site, + site: channelSite || site, }; } diff --git a/src/scrapers/private.js b/src/scrapers/private.js index 8bbeac2d..a2b5a23f 100644 --- a/src/scrapers/private.js +++ b/src/scrapers/private.js @@ -1,6 +1,6 @@ 'use strict'; -/* eslint-disable */ +/* eslint-disable newline-per-chained-call */ const bhttp = require('bhttp'); const cheerio = require('cheerio'); const moment = require('moment'); @@ -44,7 +44,7 @@ async function scrapeScene(html, url, site) { const shootId = url.split('/').slice(-1)[0]; const title = $('.video-wrapper meta[itemprop="name"]').attr('content'); - const date = moment.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate() + const date = moment.utc($('.video-wrapper meta[itemprop="uploadDate"]').attr('content'), 'MM/DD/YYYY').toDate(); const actors = $('.content-wrapper .scene-models-list a').map((actorIndex, actorElement) => $(actorElement).text()).toArray(); const description = $('.video-wrapper meta[itemprop="description"]').attr('content'); diff --git a/src/scrapers/template.js b/src/scrapers/template.js index 8cb0dc58..05c588ec 100644 --- a/src/scrapers/template.js +++ b/src/scrapers/template.js @@ -1,6 +1,6 @@ 'use strict'; -/* eslint-disable */ +/* eslint-disable newline-per-chained-call */ const bhttp = require('bhttp'); const cheerio = require('cheerio'); const moment = require('moment'); @@ -68,6 +68,7 @@ async function scrapeScene(html, url, site) { url, shootId, title, + description, actors, director, date, diff --git a/src/scrapers/vixen.js b/src/scrapers/vixen.js index 5349e469..1b551039 100644 --- a/src/scrapers/vixen.js +++ b/src/scrapers/vixen.js @@ -1,6 +1,6 @@ 'use strict'; -/* eslint-disable */ +/* eslint-disable newline-per-chained-call */ const bhttp = require('bhttp'); const cheerio = require('cheerio'); const moment = require('moment'); @@ -15,7 +15,7 @@ function scrapeLatest(html, site) { return scenes.map((scene) => { const shootId = String(scene.newId); - const title = scene.title; + const { title } = scene; const url = `${site.url}${scene.targetUrl}`; const date = moment.utc(scene.releaseDateFormatted, 'MMMM DD, YYYY').toDate(); const actors = scene.models; @@ -45,7 +45,7 @@ async function scrapeScene(html, url, site) { const scene = data.page.data[`${pathname}${search}`].data.video; const shootId = String(scene.newId); - const title = scene.title; + const { title } = scene; const date = new Date(scene.releaseDate); const actors = scene.models; const stars = scene.totalRateVal;