boiii
/
traxxx
forked from DebaucheryLibrarian/traxxx
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Updated Score scraper to accept site and with-releases argument.

This commit is contained in:
ThePendulum 2020-02-25 22:32:13 +01:00
parent 82e8ce432b
commit 646ff064a7
3 changed files with 11792 additions and 14511 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26286
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

1
package.json
View File

@ -92,6 +92,7 @@
"longjohn": "^0.2.12", "longjohn": "^0.2.12",
"mime": "^2.4.4", "mime": "^2.4.4",
"moment": "^2.24.0", "moment": "^2.24.0",
"opencv4nodejs": "^5.5.0",
"opn": "^5.5.0", "opn": "^5.5.0",
"pg": "^7.18.1", "pg": "^7.18.1",
"postgraphile": "^4.5.5", "postgraphile": "^4.5.5",

16
src/scrapers/score.js
View File

@ -149,7 +149,7 @@ async function fetchActorReleases(url, accReleases = []) {
return null; return null;
} }
async function scrapeProfile(html, actorUrl) { async function scrapeProfile(html, actorUrl, withReleases) {
const { q, qa, qi } = ex(html, '#model-page'); const { q, qa, qi } = ex(html, '#model-page');
const profile = { gender: 'female' }; const profile = { gender: 'female' };
@ -187,8 +187,10 @@ async function scrapeProfile(html, actorUrl) {
const avatar = qi('img'); const avatar = qi('img');
if (avatar) profile.avatar = avatar; if (avatar) profile.avatar = avatar;
const { origin, pathname } = new URL(actorUrl); if (withReleases) {
profile.releases = await fetchActorReleases(`${origin}${pathname}/scenes?page=1`); const { origin, pathname } = new URL(actorUrl);
profile.releases = await fetchActorReleases(`${origin}${pathname}/scenes?page=1`);
}
return profile; return profile;
} }
@ -215,7 +217,7 @@ async function fetchScene(url, site) {
return null; return null;
} }
async function fetchProfile(actorName, scraperSlug, page = 1, source = 0) { async function fetchProfile(actorName, scraperSlug, site, withReleases, page = 1, source = 0) {
const letter = actorName.charAt(0).toUpperCase(); const letter = actorName.charAt(0).toUpperCase();
const sources = [ const sources = [
@ -236,17 +238,17 @@ async function fetchProfile(actorName, scraperSlug, page = 1, source = 0) {
const actorRes = await bhttp.get(actorUrl); const actorRes = await bhttp.get(actorUrl);
if (actorRes.statusCode === 200) { if (actorRes.statusCode === 200) {
return scrapeProfile(actorRes.body.toString(), actorUrl); return scrapeProfile(actorRes.body.toString(), actorUrl, withReleases);
} }
return null; return null;
} }
return fetchProfile(actorName, scraperSlug, page + 1, source); return fetchProfile(actorName, scraperSlug, site, withReleases, page + 1, source);
} }
if (sources[source + 1]) { if (sources[source + 1]) {
return fetchProfile(actorName, scraperSlug, 1, source + 1); return fetchProfile(actorName, scraperSlug, site, withReleases, 1, source + 1);
} }
return null; return null;