DebaucheryLibrarian
/
traxxx
2
1
Fork 1
Code Issues 36 Pull Requests 1 Releases Wiki Activity

Fixed Bang scraper.

This commit is contained in:
DebaucheryLibrarian 2021-02-26 02:52:39 +01:00
parent 16a5d92efe
commit 20129eca5d
2 changed files with 10 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
migrations/20190325001339_releases.js
View File

@ -1001,7 +1001,8 @@ exports.up = knex => Promise.resolve()
// allow vim fold
return knex.raw(`
/* We need both the release entries and their search ranking, and PostGraphile does not seem to allow virtual foreign keys on function results.
* Using a view as a proxy for the search results allows us to get both a reference to the releases table, and the ranking */
* Using a view as a proxy for the search results allows us to get both a reference to the releases table, and the ranking.
* A composite type does not seem to be compatible with PostGraphile's @sortable. */
CREATE VIEW releases_search_results AS
SELECT NULL::integer as id, NULL::real as rank;

16
src/scrapers/bang.js
View File

@ -83,7 +83,7 @@ async function scrapeScene(scene, entity, options) {
release.poster = getScreenUrl(poster, scene);
release.photos = remainingScreens.map(photo => getScreenUrl(photo, scene));
if (options.includePhotos) {
if (options?.includePhotos) {
const photos = await fetchPhotos(scene);
if (photos?.length > 0) {
@ -100,11 +100,11 @@ async function scrapeScene(scene, entity, options) {
return release;
}
function scrapeAll(scenes) {
return scenes.map(({ _source: scene }) => scrapeScene(scene));
function scrapeAll(scenes, entity) {
return scenes.map(({ _source: scene }) => scrapeScene(scene, entity));
}
async function fetchActorReleases(actor) {
async function fetchActorReleases(actor, entity) {
const res = await http.post(`https://${clusterId}.us-east-1.aws.found.io/videos/video/_search`, {
size: 50,
query: {
@ -158,10 +158,10 @@ async function fetchActorReleases(actor) {
},
});
return scrapeAll(res.body.hits.hits);
return scrapeAll(res.body.hits.hits, entity);
}
async function scrapeProfile(actor, include) {
async function scrapeProfile(actor, entity, include) {
const profile = {};
profile.aliases = actor.aliases;
@ -193,7 +193,7 @@ async function scrapeProfile(actor, include) {
if (actor.image) profile.avatar = `https://i.bang.com/pornstars/${actor.identifier}.jpg`;
if (include.releases) {
profile.releases = await fetchActorReleases(actor);
profile.releases = await fetchActorReleases(actor, entity);
}
return profile;
@ -402,7 +402,7 @@ async function fetchProfile({ name: actorName }, context, include) {
const actor = res.body.hits.hits.find(hit => hit._source.name.toLowerCase() === actorName.toLowerCase());
if (actor) {
return scrapeProfile(actor._source, include);
return scrapeProfile(actor._source, context.entity, include);
}
return null;