Added MindGeek profile scraper for all MG sites.

2020-01-27 22:54:14 +01:00
parent 24fe61e064
commit 6d4fd5fd77
11 changed files with 124 additions and 23 deletions
--- a/src/scrapers/ddfnetwork.js
+++ b/src/scrapers/ddfnetwork.js
@@ -148,7 +148,9 @@ async function scrapeProfile(html, _url, actorName) {
 }

 async function fetchLatest(site, page = 1) {
-    const res = await bhttp.get(`https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`);
+    const url = `https://ddfnetwork.com/videos/search/latest/ever/${new URL(site.url).hostname}/-/${page}`;
+    console.log(url);
+    const res = await bhttp.get(url);

    return scrapeLatest(res.body.toString(), site);
 }